feat(18): adopt the shared CLI helpers in the rdkit Jobs

Dockerfile-mordred

@@ -13,8 +13,8 @@ WORKDIR ${HOME}
 # install our requirements. RDKit is present in base container
 RUN pip install mordredcommunity==2.0.4 \
   im-standardize-molecule==0.1.0 \
-  im-data-manager-job-utilities==1.3.0 \
-  im-rdkit-utilities==1.0.0
+  im-data-manager-job-utilities>=1.4.0 \
+  im-rdkit-utilities>=1.1.2
 
 COPY im_mordred/descriptor_generator.py ./im_mordred/
 COPY utils.py ./

Dockerfile-prep

@@ -23,8 +23,8 @@ WORKDIR ${HOME}
 RUN pip install scikit-learn==1.2.2\
   pdb2pqr==3.6.1\
   im-standardize-molecule==0.1.0\
-  im-data-manager-job-utilities==1.3.0\
-  im-rdkit-utilities==1.0.0\
+  im-data-manager-job-utilities>=1.4.0\
+  im-rdkit-utilities>=1.1.2\
   sigfig==1.3.19
 
 COPY *.py fpscores.pkl.gz site_substructures.smarts ./

cluster_butina.py

@@ -235,22 +235,10 @@ def main():
 
     # command line args definitions #########################################
     parser = argparse.ArgumentParser(description='RDKit Butina Cluster')
-    parser.add_argument('-i', '--input', required=True, help="File with molecules to cluster (.sdf or .smi)")
-    parser.add_argument('-o', '--output', required=True, help="Output file (.sdf or .smi)")
+    rdkit_utils.add_common_molecule_io_args(parser, output_required=True)
 
-    parser.add_argument('-k', '--omit-fields', action='store_true',
-                        help="Don't include fields from the input in the output")
 
     # to pass tab as the delimiter specify it as $'\t' or use one of the symbolic names 'comma', 'tab', 'space' or 'pipe'
-    parser.add_argument('-d', '--delimiter', help="Delimiter when using SMILES")
-    parser.add_argument('--id-column', help="Column for name field (zero based integer for .smi, text for SDF)")
-    parser.add_argument('--mol-column', type=int, default=0,
-                        help="Column index for molecule when using delineated text formats (zero based integer)")
-    parser.add_argument('--read-header', action='store_true',
-                        help="Read a header line with the field names when reading .smi or .txt")
-    parser.add_argument('--write-header', action='store_true', help='Write a header line when writing .smi or .txt')
-    parser.add_argument('--read-records', default=100, type=int,
-                        help="Read this many records to determine the fields that are present")
 
     parser.add_argument('-t', '--threshold', type=float, default=0.7,
                         help='similarity clustering threshold (1.0 means identical)')

enumerate.py

@@ -27,6 +27,7 @@
 """
 import os, sys, argparse, traceback, uuid, gzip
 import utils, rdkit_utils
+from dm_job_utilities.cli import ProgressReporter, add_reporting_args
 from dm_job_utilities.dm_log import DmLog
 
 from rdkit import Chem
@@ -104,6 +105,8 @@ def execute(input, output, delimiter=' ',
             max_tautomers=25,
             interval=0):
 
+    reporter = ProgressReporter(interval)
+
     DmLog.emit_event('Executing ...')
 
     if enumerate_tautomers or combinatorial:
@@ -136,8 +139,7 @@ def execute(input, output, delimiter=' ',
 
             count += 1
 
-            if interval and count % interval == 0:
-                DmLog.emit_event("Processed {} records".format(count))
+            reporter.report(count)
 
             if fragment != 'none':
                 mol = rdkit_utils.fragment(mol, fragment)
@@ -229,16 +231,8 @@ def main():
     ### command line args definitions #########################################
 
     parser = argparse.ArgumentParser(description='Enumerate candidates')
-    parser.add_argument('-i', '--input', required=True, help="Input file (.smi, .sdf)")
-    parser.add_argument('-o', '--output', required=True, help="Output file (.sdf)")
-    parser.add_argument('-d', '--delimiter', default='\t', help="Delimiter")
-    parser.add_argument('--id-column', help="Column for name field (zero based integer for .smi, text for SDF)")
-    parser.add_argument('--mol-column', type=int, default=0,
-                        help="Column index for molecule when using delineated text formats (zero based integer)")
-    parser.add_argument('--read-header', action='store_true',
-                        help="Read a header line with the field names when reading .smi or .txt")
-    parser.add_argument('--read-records', default=100, type=int,
-                        help="Read this many records to determine the fields that are present")
+    rdkit_utils.add_common_molecule_io_args(parser, output_required=True)
+    add_reporting_args(parser)
     parser.add_argument('-f', '--fragment-method', choices=['hac', 'mw', 'none'], default='hac',
                         help='Strategy for picking largest fragment (mw or hac or none')
     parser.add_argument('--enumerate-charges', help='Enumerate charge forms', action='store_true')
@@ -255,7 +249,6 @@ def main():
     parser.add_argument('--add-hydrogens', action='store_true', help='Include hydrogens in the output')
     parser.add_argument('--try-embedding', action='store_true', help='Try to embed a 3D molecule to verify the stereochemisty is sane')
     parser.add_argument("--max-tautomers", type=int, default=25, help="Maximum number of tautomers to generate")
-    parser.add_argument("--interval", type=int, help="Reporting interval")
 
     args = parser.parse_args()
     DmLog.emit_event("enumerate_candidates: ", args)

im_mordred/descriptor_generator.py

@@ -21,6 +21,7 @@
 from rdkit import Chem
 
 import utils, rdkit_utils
+from dm_job_utilities.cli import ProgressReporter, add_reporting_args
 from dm_job_utilities.dm_log import DmLog
 
 
@@ -37,6 +38,8 @@ def run(input,
         read_records=50,
         interval=1000):
 
+    reporter = ProgressReporter(interval)
+
     if include_3d:
         if not (input.endswith('.sdf') or input.endswith('.sdf.gz')):
             DmLog.emit_event('If calculating 3D descriptors input must be a SDF with 3D molecules')
@@ -97,8 +100,7 @@ def run(input,
 
         count += 1
 
-        if interval and count % interval == 0:
-            DmLog.emit_event("Processed {} records".format(count))
+        reporter.report(count)
         if count % 50000 == 0:
             # Emit a 'total' cost, replacing all prior costs
             DmLog.emit_cost(count)
@@ -150,29 +152,17 @@ def main():
     # ----- command line args definitions ---------------------------------------------
 
     parser = argparse.ArgumentParser(description='Mordred 2D descriptors')
-    parser.add_argument('-i', '--input', required=True, help="Input file (.smi or .sdf)")
-    parser.add_argument('-o', '--output', default='descriptors2d.smi', help="Output file (.smi or .sdf")
+    rdkit_utils.add_common_molecule_io_args(parser, output_default="descriptors2d.smi")
+    add_reporting_args(parser)
 
     parser.add_argument('-f', '--fragment-method', choices=['hac', 'mw', 'none'], default='hac',
                         help='Strategy for picking largest fragment (mw or hac or none')
 
     parser.add_argument('-3', '--include-3d', action='store_true',
                         help="Include 3D descriptors (requires 3D molecules in SDF file)")
 
-    parser.add_argument('-k', '--omit-fields', action='store_true',
-                        help="Don't include fields from the input in the output")
 
     # to pass tab as the delimiter specify it as $'\t' or use one of the symbolic names 'comma', 'tab', 'space' or 'pipe'
-    parser.add_argument('-d', '--delimiter', help="Delimiter when using SMILES")
-    parser.add_argument('--id-column', help="Column for name field (zero based integer for .smi, text for SDF)")
-    parser.add_argument('--mol-column', type=int, default=0,
-                        help="Column index for molecule when using delineated text formats (zero based integer)")
-    parser.add_argument('--read-header', action='store_true',
-                        help="Read a header line with the field names when reading .smi or .txt")
-    parser.add_argument('--write-header', action='store_true', help='Write a header line when writing .smi or .txt')
-    parser.add_argument('--read-records', default=100, type=int,
-                        help="Read this many records to determine the fields that are present")
-    parser.add_argument("--interval", type=int, help="Reporting interval")
 
     args = parser.parse_args()
     DmLog.emit_event("descriptor_calc: ", args)

le_conformers.py

@@ -24,6 +24,7 @@
 import os, sys, argparse, traceback, time, gzip
 
 import utils, rdkit_utils
+from dm_job_utilities.cli import ProgressReporter, add_reporting_args
 from dm_job_utilities.dm_log import DmLog
 
 from rdkit import Chem
@@ -114,6 +115,8 @@ def execute(input, output, minimize_cycles=500, fragment='hac',
             delimiter=' ', id_column=None, mol_column=0, read_records=100, read_header=False,
             remove_hydrogens=False, rms_threshold=1.0, num_conformers=None, interval=None):
 
+    reporter = ProgressReporter(interval)
+
     DmLog.emit_event('Executing ...')
 
     utils.expand_path(output)
@@ -137,8 +140,7 @@ def execute(input, output, minimize_cycles=500, fragment='hac',
 
                 input_count += 1
 
-                if interval and input_count % interval == 0:
-                    DmLog.emit_event("Processed {} records".format(input_count))
+                reporter.report(input_count)
 
                 if not mol:
                     error_count += 1
@@ -216,17 +218,9 @@ def main():
     ### command line args definitions #########################################
 
     parser = argparse.ArgumentParser(description='Enumerate conformers')
-    parser.add_argument('-i', '--input', required=True, help="Input file (.smi, .sdf)")
-    parser.add_argument('-o', '--output', required=True, help="Output file (.sdf)")
-
-    parser.add_argument('-d', '--delimiter', help="Delimiter when using SMILES")
-    parser.add_argument('--id-column', help="Column for name field (zero based integer for .smi, text for SDF)")
-    parser.add_argument('--mol-column', type=int, default=0,
-                        help="Column index for molecule when using delineated text formats (zero based integer)")
-    parser.add_argument('--read-header', action='store_true',
-                        help="Read a header line with the field names when reading .smi or .txt")
-    parser.add_argument('--read-records', default=100, type=int,
-                        help="Read this many records to determine the fields that are present")
+    rdkit_utils.add_common_molecule_io_args(parser, output_required=True)
+    add_reporting_args(parser)
+
 
     parser.add_argument('-f', '--fragment-method', choices=['hac', 'mw', 'none'], default='hac',
                         help='Strategy for picking largest fragment (mw or hac or none')
@@ -236,7 +230,6 @@ def main():
     parser.add_argument('-t', '--rms-threshold', type=float, default=1.0, help="RMS threshold for excluding conformers")
     parser.add_argument('--remove-hydrogens', action='store_true', help='Remove hydrogens from the output')
 
-    parser.add_argument("--interval", type=int, help="Reporting interval")
 
     args = parser.parse_args()
     DmLog.emit_event("le_conformers: ", args)

rdkit_dedup.py

@@ -19,6 +19,7 @@
 import traceback
 
 import utils, rdkit_utils
+from dm_job_utilities.cli import ProgressReporter, add_reporting_args
 from dm_job_utilities.dm_log import DmLog
 
 from rdkit import Chem
@@ -38,6 +39,8 @@ def process(input,
             max_hac=None,
             interval=0):
 
+    reporter = ProgressReporter(interval)
+
     utils.expand_path(outfile)
 
     count = 0
@@ -83,8 +86,7 @@ def process(input,
 
         count += 1
 
-        if interval and count % interval == 0:
-            DmLog.emit_event("Processed {} records".format(count))
+        reporter.report(count)
         if count % 50000 == 0:
             # Emit a 'total' cost, replacing all prior costs
             DmLog.emit_cost(count)
@@ -148,26 +150,14 @@ def main():
     ### command line args definitions #########################################
 
     parser = argparse.ArgumentParser(description='RDKit deduplicate')
-    parser.add_argument('-i', '--input', required=True, help="Input file as SMILES or SDF")
-    parser.add_argument('-o', '--outfile', required=True, help="Output file as SMILES or SDF")
-    parser.add_argument('-k', '--omit-fields', action='store_true',
-                        help="Don't include fields from the input in the output")
+    rdkit_utils.add_common_molecule_io_args(parser, output_required=True)
+    add_reporting_args(parser)
     # to pass tab as the delimiter specify it as $'\t' or use one of the symbolic names 'comma', 'tab', 'space' or 'pipe'
-    parser.add_argument('-d', '--delimiter', help="Delimiter when using SMILES")
-    parser.add_argument('--id-column', help="Column for name field (zero based integer for .smi, text for SDF)")
-    parser.add_argument('--mol-column', type=int, default=0,
-                        help="Column index for molecule when using delineated text formats (zero based integer)")
-    parser.add_argument('--read-header', action='store_true',
-                        help="Read a header line with the field names when reading .smi or .txt")
-    parser.add_argument('--write-header', action='store_true', help='Write a header line when writing .smi or .txt')
-    parser.add_argument('--read-records', default=100, type=int,
-                        help="Read this many records to determine the fields that are present")
     parser.add_argument('-m', '--mode', choices=['hac', 'mw', 'none'], default='hac',
                         help='Strategy for picking largest fragment (mw or hac or none')
     parser.add_argument("--min-hac", type=int, help="Minimum heavy atom count to consider")
     parser.add_argument("--max-hac", type=int, help="Maximum heavy atom count to consider")
 
-    parser.add_argument("--interval", type=int, help="Reporting interval")
 
     args = parser.parse_args()
     DmLog.emit_event("rdk_dedup.py: ", args)
@@ -177,7 +167,7 @@ def main():
 
     t0 = time.time()
     count, errors, duplicates, excluded = process(
-        args.input, args.outfile, mode=args.mode, delimiter=delimiter,
+        args.input, args.output, mode=args.mode, delimiter=delimiter,
         id_column=args.id_column, mol_column=args.mol_column, omit_fields=args.omit_fields,
         read_header=args.read_header, write_header=args.write_header,
         read_records=args.read_records,

rdkit_props.py

@@ -18,6 +18,7 @@
 import argparse, os, gzip, time
 
 import utils, rdkit_utils
+from dm_job_utilities.cli import ProgressReporter, add_reporting_args
 from dm_job_utilities.dm_log import DmLog
 
 from rdkit import Chem
@@ -48,6 +49,8 @@ def process(input,
             read_records=100,
             interval=0):
 
+    reporter = ProgressReporter(interval)
+
     utils.log('Using calculations:', calcs)
 
     utils.expand_path(outfile)
@@ -96,8 +99,7 @@ def process(input,
 
         count += 1
 
-        if interval and count % interval == 0:
-            DmLog.emit_event("Processed {} records".format(count))
+        reporter.report(count)
         if count % 50000 == 0:
             # Emit a 'total' cost, replacing all prior costs
             DmLog.emit_cost(count)
@@ -166,28 +168,16 @@ def main():
     ### command line args definitions #########################################
 
     parser = argparse.ArgumentParser(description='Calc RDKit props')
-    parser.add_argument('-i', '--input', required=True, help="Input file as SMILES or SDF")
-    parser.add_argument('-o', '--outfile', required=True, help="Output file as SMILES or SDF")
+    rdkit_utils.add_common_molecule_io_args(parser, output_required=True)
+    add_reporting_args(parser)
 
-    parser.add_argument('-k', '--omit-fields', action='store_true',
-                        help="Don't include fields from the input in the output")
 
     # to pass tab as the delimiter specify it as $'\t' or use one of the symbolic names 'comma', 'tab', 'space' or 'pipe'
-    parser.add_argument('-d', '--delimiter', help="Delimiter when using SMILES")
-    parser.add_argument('--id-column', help="Column for name field (zero based integer for .smi, text for SDF)")
-    parser.add_argument('--mol-column', type=int, default=0,
-                        help="Column index for molecule when using delineated text formats (zero based integer)")
-    parser.add_argument('--read-header', action='store_true',
-                        help="Read a header line with the field names when reading .smi or .txt")
-    parser.add_argument('--write-header', action='store_true', help='Write a header line when writing .smi or .txt')
-    parser.add_argument('--read-records', default=100, type=int,
-                        help="Read this many records to determine the fields that are present")
 
     parser.add_argument('-a', '--all', action='store_true', help="Calculate all properties")
     for key in calc_props:
         parser.add_argument('--' + key.replace('_', '-'), action='store_true', help=calc_props[key][1])
 
-    parser.add_argument("--interval", type=int, help="Reporting interval")
 
     args = parser.parse_args()
     DmLog.emit_event("rdk_props.py: ", args)
@@ -206,7 +196,7 @@ def main():
 
     t0 = time.time()
     count, errors = process(
-        args.input, args.outfile, calcs_to_use, delimiter, id_column=args.id_column, mol_column=args.mol_column,
+        args.input, args.output, calcs_to_use, delimiter, id_column=args.id_column, mol_column=args.mol_column,
         omit_fields=args.omit_fields, read_header=args.read_header, write_header=args.write_header,
         read_records=args.read_records, interval=args.interval, )
     t1 = time.time()