Merge pull request biocore#221 from jdereus/issue214

Prep work for issues 195 and 214: "notes" field on Process

labman/db/plate.py

@@ -380,7 +380,7 @@ def process(self):
 
     @property
     def quantification_process(self):
-        """The quantification process of the plate
+        """The quantification process(es) of the plate
 
         Returns
         -------

labman/db/process.py

@@ -84,9 +84,10 @@ def factory(process_id):
         return instance
 
     @classmethod
-    def _common_creation_steps(cls, user, process_date=None):
+    def _common_creation_steps(cls, user, process_date=None, notes=None):
         if process_date is None:
             process_date = date.today()
+
         with sql_connection.TRN as TRN:
             sql = """SELECT process_type_id
                      FROM qiita.process_type
@@ -95,10 +96,10 @@ def _common_creation_steps(cls, user, process_date=None):
             pt_id = TRN.execute_fetchlast()
 
             sql = """INSERT INTO qiita.process
-                        (process_type_id, run_date, run_personnel_id)
-                     VALUES (%s, %s, %s)
+                        (process_type_id, run_date, run_personnel_id, notes)
+                     VALUES (%s, %s, %s, %s)
                      RETURNING process_id"""
-            TRN.add(sql, [pt_id, process_date, user.id])
+            TRN.add(sql, [pt_id, process_date, user.id, notes])
             p_id = TRN.execute_fetchlast()
         return p_id
 
@@ -132,6 +133,10 @@ def date(self):
     def personnel(self):
         return user_module.User(self._get_process_attr('run_personnel_id'))
 
+    @property
+    def notes(self):
+        return self._get_process_attr('notes')
+
     @property
     def process_id(self):
         return self._get_process_attr('process_id')
@@ -160,13 +165,14 @@ class _Process(Process):
     """Process object
 
     Not all processes have a specific subtable, so we need to override the
-    date and personnel attributes
+    date, personnel, and notes attributes
 
     Attributes
     ----------
     id
     date
     personnel
+    notes
     """
     _table = 'qiita.process'
     _id_column = 'process_id'
@@ -179,6 +185,10 @@ def date(self):
     def personnel(self):
         return user_module.User(self._get_attr('run_personnel_id'))
 
+    @property
+    def notes(self):
+        return self._get_attr('notes')
+
     @property
     def process_id(self):
         return self._get_attr('process_id')
@@ -302,7 +312,7 @@ def create(cls, user, external_id, volume, reagent_type):
 
         Returns
         -------
-        ReagentCreationProce
+        ReagentCreationProcess
         """
         with sql_connection.TRN:
             # Add the row to the process table
@@ -1682,47 +1692,30 @@ def parse(contents, file_format="minipico", rows=8, cols=12):
         return array.astype(float)
 
     @classmethod
-    def create_manual(cls, user, quantifications):
-        """Creates a new manual quantification process
+    def create_manual(cls, user, quantifications, notes=None):
+        """Creates a new quantification process for a pool
 
         Parameters
         ----------
         user: labman.db.user.User
             User performing the quantification process
         quantifications: list of dict
             The quantifications in the form of {'composition': Composition,
-            'conenctration': float}
+            'concentration': float}
+        notes: str
+            Description of the quantification process
+                (e.g., 'Requantification of failed plate', etc).
+                Default: None
 
         Returns
         -------
         QuantificationProcess
         """
-        with sql_connection.TRN as TRN:
-            # Add the row to the process table
-            process_id = cls._common_creation_steps(user)
-
-            # Add the row to the quantification process table
-            sql = """INSERT INTO qiita.quantification_process (process_id)
-                     VALUES (%s) RETURNING quantification_process_id"""
-            TRN.add(sql, [process_id])
-            instance = cls(TRN.execute_fetchlast())
-
-            sql = """INSERT INTO qiita.concentration_calculation
-                        (quantitated_composition_id, upstream_process_id,
-                         raw_concentration)
-                     VALUES (%s, %s, %s)"""
-            sql_args = []
-            for quant in quantifications:
-                sql_args.append([quant['composition'].composition_id,
-                                 instance.id, quant['concentration']])
-
-            TRN.add(sql, sql_args, many=True)
-            TRN.execute()
-        return instance
+        return cls._create(user, notes, quantifications)
 
     @classmethod
-    def create(cls, user, plate, concentrations):
-        """Creates a new quantification process
+    def create(cls, user, plate, concentrations, notes=None):
+        """Creates a new quantification process for a plate
 
         Parameters
         ----------
@@ -1732,14 +1725,43 @@ def create(cls, user, plate, concentrations):
             The plate being quantified
         concentrations: 2D np.array
             The plate concentrations
+        notes: str
+            Description of the quantification process
+                (e.g., 'Requantification of failed plate', etc).
+                Default: None
+
+        Returns
+        -------
+        QuantificationProcess
+        """
+        return cls._create(user, notes, concentrations, plate)
+
+    @classmethod
+    def _create(cls, user, notes, concentrations, plate=None):
+        """Creates a new quantification process for a plate or a pool.
+
+        Parameters
+        ----------
+        user: labman.db.user.User
+            User performing the quantification process
+        notes: str
+            Description of the quantification process
+                (e.g., 'Requantification of failed plate', etc).  May be None.
+        concentrations: 2D np.array OR list of dict
+            If plate is not None, the plate concentrations as a 2D np.array.
+            If plate IS None, the pool component concentrations as a list of
+                dicts where each dict is in the form of
+                {'composition': Composition,  'concentration': float}
+        plate: labman.db.plate.Plate
+            The plate being quantified, if relevant. Default: None
 
         Returns
         -------
         QuantificationProcess
         """
         with sql_connection.TRN as TRN:
             # Add the row to the process table
-            process_id = cls._common_creation_steps(user)
+            process_id = cls._common_creation_steps(user, notes=notes)
 
             # Add the row to the quantification process table
             sql = """INSERT INTO qiita.quantification_process (process_id)
@@ -1751,22 +1773,44 @@ def create(cls, user, plate, concentrations):
                         (quantitated_composition_id, upstream_process_id,
                          raw_concentration)
                      VALUES (%s, %s, %s)"""
-            sql_args = []
-            layout = plate.layout
 
-            for p_row, c_row in zip(layout, concentrations):
-                for well, conc in zip(p_row, c_row):
-                    if well is not None:
-                        sql_args.append([well.composition.composition_id,
-                                         instance.id, conc])
+            if plate is not None:
+                sql_args = cls._generate_concentration_inputs_for_plate(
+                    plate, concentrations, instance)
+            else:
+                sql_args = cls._generate_concentration_inputs_for_pool(
+                    concentrations, instance)
 
             if len(sql_args) == 0:
                 raise ValueError('No concentration values have been provided')
 
             TRN.add(sql, sql_args, many=True)
             TRN.execute()
 
-            return instance
+        return instance
+
+    @classmethod
+    def _generate_concentration_inputs_for_plate(cls, plate, concentrations,
+                                                 quant_process_instance):
+        sql_args = []
+        layout = plate.layout
+
+        for p_row, c_row in zip(layout, concentrations):
+            for well, conc in zip(p_row, c_row):
+                if well is not None:
+                    sql_args.append([well.composition.composition_id,
+                                     quant_process_instance.id, conc])
+        return sql_args
+
+    @classmethod
+    def _generate_concentration_inputs_for_pool(cls, concentrations,
+                                                quant_process_instance):
+        sql_args = []
+        for quant in concentrations:
+            sql_args.append([quant['composition'].composition_id,
+                             quant_process_instance.id,
+                             quant['concentration']])
+        return sql_args
 
     @property
     def concentrations(self):

labman/db/support_files/db_patch.sql

@@ -127,6 +127,7 @@ CREATE TABLE qiita.process (
 	process_type_id      integer  NOT NULL,
 	run_date             date  NOT NULL,
 	run_personnel_id     varchar  NOT NULL,
+	notes                varchar(600)  ,
 	CONSTRAINT pk_process PRIMARY KEY ( process_id )
  );
 
@@ -344,12 +345,12 @@ CREATE TABLE qiita.sequencing_process_lanes (
 	sequencing_process_id bigint  NOT NULL,
 	pool_composition_id  bigint  NOT NULL,
 	lane_number          integer  NOT NULL,
-	CONSTRAINT idx_sequencinc_process_lanes_0 UNIQUE ( sequencing_process_id, pool_composition_id, lane_number )
+	CONSTRAINT idx_sequencing_process_lanes_0 UNIQUE ( sequencing_process_id, pool_composition_id, lane_number )
  );
 
-CREATE INDEX idx_sequencinc_process_lanes_1 ON qiita.sequencing_process_lanes ( sequencing_process_id );
+CREATE INDEX idx_sequencing_process_lanes_1 ON qiita.sequencing_process_lanes ( sequencing_process_id );
 
-CREATE INDEX idx_sequencinc_process_lanes_2 ON qiita.sequencing_process_lanes ( pool_composition_id );
+CREATE INDEX idx_sequencing_process_lanes_2 ON qiita.sequencing_process_lanes ( pool_composition_id );
 
 CREATE TABLE qiita.shotgun_combo_primer_set (
 	shotgun_combo_primer_set_id bigserial  NOT NULL,
@@ -645,9 +646,9 @@ ALTER TABLE qiita.sequencing_process ADD CONSTRAINT fk_sequencing_process_eq FOR
 
 ALTER TABLE qiita.sequencing_process_contacts ADD CONSTRAINT fk_sequencing_process_contacts FOREIGN KEY ( sequencing_process_id ) REFERENCES qiita.sequencing_process( sequencing_process_id );
 
-ALTER TABLE qiita.sequencing_process_lanes ADD CONSTRAINT fk_sequencinc_process_lanes FOREIGN KEY ( sequencing_process_id ) REFERENCES qiita.sequencing_process( sequencing_process_id );
+ALTER TABLE qiita.sequencing_process_lanes ADD CONSTRAINT fk_sequencing_process_lanes FOREIGN KEY ( sequencing_process_id ) REFERENCES qiita.sequencing_process( sequencing_process_id );
 
-ALTER TABLE qiita.sequencing_process_lanes ADD CONSTRAINT fk_sequencinc_process_lanes_pool FOREIGN KEY ( pool_composition_id ) REFERENCES qiita.pool_composition( pool_composition_id );
+ALTER TABLE qiita.sequencing_process_lanes ADD CONSTRAINT fk_sequencing_process_lanes_pool FOREIGN KEY ( pool_composition_id ) REFERENCES qiita.pool_composition( pool_composition_id );
 
 ALTER TABLE qiita.shotgun_combo_primer_set ADD CONSTRAINT fk_shotgun_combo_primer_set_i5 FOREIGN KEY ( i5_primer_set_composition_id ) REFERENCES qiita.primer_set_composition( primer_set_composition_id );
 

labman/db/support_files/labman.dbs

@@ -774,21 +774,21 @@ Nullable for case where pooling is done manually; then the person who did it is
 			<column name="sequencing_process_id" type="bigint" jt="-5" mandatory="y" />
 			<column name="pool_composition_id" type="bigint" jt="-5" mandatory="y" />
 			<column name="lane_number" type="integer" jt="4" mandatory="y" />
-			<index name="idx_sequencinc_process_lanes_1" unique="NORMAL" >
+			<index name="idx_sequencing_process_lanes_1" unique="NORMAL" >
 				<column name="sequencing_process_id" />
 			</index>
-			<index name="idx_sequencinc_process_lanes_2" unique="NORMAL" >
+			<index name="idx_sequencing_process_lanes_2" unique="NORMAL" >
 				<column name="pool_composition_id" />
 			</index>
-			<index name="idx_sequencinc_process_lanes_0" unique="UNIQUE" >
+			<index name="idx_sequencing_process_lanes_0" unique="UNIQUE" >
 				<column name="sequencing_process_id" />
 				<column name="pool_composition_id" />
 				<column name="lane_number" />
 			</index>
-			<fk name="fk_sequencinc_process_lanes" to_schema="qiita" to_table="sequencing_process" >
+			<fk name="fk_sequencing_process_lanes" to_schema="qiita" to_table="sequencing_process" >
 				<fk_column name="sequencing_process_id" pk="sequencing_process_id" />
 			</fk>
-			<fk name="fk_sequencinc_process_lanes_pool" to_schema="qiita" to_table="pool_composition" >
+			<fk name="fk_sequencing_process_lanes_pool" to_schema="qiita" to_table="pool_composition" >
 				<fk_column name="pool_composition_id" pk="pool_composition_id" />
 			</fk>
 		</table>

labman/db/support_files/labman.html

@@ -456,11 +456,11 @@
 	sequencing_process references equipment ( sequencer_id -> equipment_id )</title>
 </path>
 <path transform='translate(7,0)' marker-start='url(#foot)'     d='M 1935 915 L 1942,915 Q 1950,915 1950,922 L 1950,1027 Q 1950,1035 1942,1035 L 1935,1035' >
-	<title>Foreign Key fk_sequencinc_process_lanes
+	<title>Foreign Key fk_sequencing_process_lanes
 	sequencing_process_lanes references sequencing_process ( sequencing_process_id )</title>
 </path>
 <path transform='translate(7,0)' marker-start='url(#foot)' marker-end='url(#arrow)'    d='M 1755 930 L 1717,930 Q 1710,930 1710,922 L 1710,427 Q 1710,420 1702,420 L 1695,420' >
-	<title>Foreign Key fk_sequencinc_process_lanes_pool
+	<title>Foreign Key fk_sequencing_process_lanes_pool
 	sequencing_process_lanes references pool_composition ( pool_composition_id )</title>
 </path>
 <path transform='translate(7,0)' marker-start='url(#foot)'     d='M 420 1065 L 412,1065 Q 405,1065 405,1057 L 405,1050' >
@@ -2603,26 +2603,26 @@
 		<td>  </td>
 	</tr>
 <tr><th colspan='3'><b>Indexes</b></th></tr>
-	<tr>		<td>idx&#095;sequencinc&#095;process&#095;lanes&#095;1 </td>
+	<tr>		<td>idx&#095;sequencing&#095;process&#095;lanes&#095;1 </td>
 		<td> ON sequencing&#095;process&#095;id</td>
 		<td>  </td>
 	</tr>
-	<tr>		<td>idx&#095;sequencinc&#095;process&#095;lanes&#095;2 </td>
+	<tr>		<td>idx&#095;sequencing&#095;process&#095;lanes&#095;2 </td>
 		<td> ON pool&#095;composition&#095;id</td>
 		<td>  </td>
 	</tr>
-	<tr>		<td>idx&#095;sequencinc&#095;process&#095;lanes&#095;0 unique</td>
+	<tr>		<td>idx&#095;sequencing&#095;process&#095;lanes&#095;0 unique</td>
 		<td> ON sequencing&#095;process&#095;id&#044; pool&#095;composition&#095;id&#044; lane&#095;number</td>
 		<td>  </td>
 	</tr>
 <tr><th colspan='3'><b>Foreign Keys</b></th></tr>
 	<tr>
-		<td>fk_sequencinc_process_lanes</td>
+		<td>fk_sequencing_process_lanes</td>
 		<td > ( sequencing&#095;process&#095;id ) ref <a href='#sequencing&#095;process'>sequencing&#095;process</a> (sequencing&#095;process&#095;id) </td>
 		<td>  </td>
 	</tr>
 	<tr>
-		<td>fk_sequencinc_process_lanes_pool</td>
+		<td>fk_sequencing_process_lanes_pool</td>
 		<td > ( pool&#095;composition&#095;id ) ref <a href='#pool&#095;composition'>pool&#095;composition</a> (pool&#095;composition&#095;id) </td>
 		<td>  </td>
 	</tr>

labman/db/support_files/populate_test_db.sql

@@ -181,6 +181,8 @@ DECLARE
     -- Variables for shotgun lib concentration
     sh_lib_quant_process_id             BIGINT;
     sh_lib_quant_subprocess_id          BIGINT;
+    sh_lib_quant_process_id2            BIGINT;
+    sh_lib_quant_subprocess_id2         BIGINT;
     sh_lib_raw_sample_conc              REAL;
     sh_lib_comp_sample_conc             REAL;
 
@@ -852,6 +854,18 @@ BEGIN
         VALUES (shotgun_sequencing_subprocess_id, '[email protected]'),
                (shotgun_sequencing_subprocess_id, '[email protected]');
 
+    --------------------------------------------
+    ---- LIBRARY QUANTIFICATION PROCESS REDO ---
+    --------------------------------------------
+    -- Putting it here at the end so a not to screw up any of the ids expected for
+    -- processes defined above.
+    INSERT INTO qiita.process (process_type_id, run_date, run_personnel_id, notes)
+        VALUES (pg_quant_process_type_id, '10/26/2017', '[email protected]', 'Requantification--oops')
+        RETURNING process_id INTO sh_lib_quant_process_id2;
+
+    INSERT INTO qiita.quantification_process (process_id)
+        VALUES (sh_lib_quant_process_id2)
+        RETURNING quantification_process_id INTO sh_lib_quant_subprocess_id2;
 
     -- Start plating samples - to make this easier, we are going to plate the
     -- same 12 samples in the first 6 rows of the plate, in the 7th row we are
@@ -1017,6 +1031,10 @@ BEGIN
                     INSERT INTO qiita.concentration_calculation (quantitated_composition_id, upstream_process_id, raw_concentration, computed_concentration)
                         VALUES (shotgun_lib_comp_id, sh_lib_quant_subprocess_id, sh_lib_raw_sample_conc, sh_lib_comp_sample_conc);
 
+                    -- Re-quantify library plate
+                    INSERT INTO qiita.concentration_calculation (quantitated_composition_id, upstream_process_id, raw_concentration, computed_concentration)
+                        VALUES (shotgun_lib_comp_id, sh_lib_quant_subprocess_id2, sh_lib_raw_sample_conc+1, sh_lib_comp_sample_conc+2);
+
                     -- Pooling
                     INSERT INTO qiita.pool_composition_components (output_pool_composition_id, input_composition_id, input_volume, percentage_of_output)
                         VALUES (sh_pool_subcomposition_id, shotgun_lib_comp_id, 1, 1/384);