From 3ea165866bf3e74cb093921c9a326bc6b35a35bc Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Tue, 20 Feb 2018 16:50:14 -0800 Subject: [PATCH 01/18] updated pooling functions to work for amplicon or shotgun pooling --- labman/db/process.py | 163 ++++++++------------------------ labman/db/tests/test_process.py | 26 ++--- 2 files changed, 56 insertions(+), 133 deletions(-) diff --git a/labman/db/process.py b/labman/db/process.py index 1b033a82..7048d687 100644 --- a/labman/db/process.py +++ b/labman/db/process.py @@ -1565,7 +1565,7 @@ class QuantificationProcess(Process): _process_type = 'quantification' @staticmethod - def _compute_shotgun_pico_concentration(dna_vals, size=500): + def _compute_pico_concentration(dna_vals, size=500): """Computes molar concentration of libraries from library DNA concentration values. @@ -1787,64 +1787,29 @@ def concentrations(self): (composition_module.Composition.factory(comp_id), r_con, c_con) for comp_id, r_con, c_con in TRN.execute_fetchindex()] - def compute_concentrations(self, dna_amount=240, min_val=1, max_val=15, - blank_volume=2, size=500): - """Compute the normalized concentrations + def compute_concentrations(self, size=500): + """Compute the normalized library molarity based on pico green dna + concentrations estimates. Parameters ---------- - dna_amount: float, optional - (Amplicon) Total amount of DNA, in ng. Default: 240 - min_val: float, optional - (Amplicon) Minimum amount of DNA to normalize to (nM). Default: 1 - max_val: float, optional - (Amplicon) Maximum value. Wells above this number will be - excluded (nM). Default: 15 - blank_volume: float, optional - (Amplicon) Amount to pool for the blanks (nM). Default: 2. size: int, optional - (Shotgun) The average library molecule size, in bp. + The average library molecule size, in bp. """ concentrations = self.concentrations layout = concentrations[0][0].container.plate.layout res = None - if isinstance(concentrations[0][0], - composition_module.LibraryPrep16SComposition): - # Amplicon - sample_concs = np.zeros_like(layout, dtype=float) - is_blank = np.zeros_like(layout, dtype=bool) - for comp, r_conc, _ in concentrations: - well = comp.container - row = well.row - 1 - col = well.column - 1 - sample_concs[row][col] = r_conc - sc = comp.gdna_composition.sample_composition - is_blank[row][col] = sc.sample_composition_type == 'blank' - - res = QuantificationProcess._compute_amplicon_pool_values( - sample_concs, dna_amount) - res[sample_concs < min_val] = min_val - # If there is any sample whose concentration is above the - # user-defined max_value, the decision is to not pool that sample. - # To not pool the sample, define it's volume to 0 and it will not - # get pooled. - res[sample_concs > max_val] = 0 - res[is_blank] = blank_volume - elif isinstance(concentrations[0][0], - composition_module.LibraryPrepShotgunComposition): - # Shotgun - sample_concs = np.zeros_like(layout, dtype=float) - for comp, r_conc, _ in concentrations: - well = comp.container - row = well.row - 1 - col = well.column - 1 - sample_concs[row][col] = r_conc - - res = QuantificationProcess._compute_shotgun_pico_concentration( - sample_concs, size) - # No need for else, because if it is not one of the above types - # we don't need to do anything + + sample_concs = np.zeros_like(layout, dtype=float) + for comp, r_conc, _ in concentrations: + well = comp.container + row = well.row - 1 + col = well.column - 1 + sample_concs[row][col] = r_conc + + res = QuantificationProcess._compute_pico_concentration( + sample_concs, size) if res is not None: sql_args = [] @@ -1927,7 +1892,7 @@ def estimate_pool_conc_vol(sample_vols, sample_concs): return (pool_conc, total_vol) @staticmethod - def compute_shotgun_pooling_values_eqvol(sample_concs, total_vol=60.0): + def compute_pooling_values_eqvol(sample_concs, total_vol=60.0): """Computes molar concentration of libraries from concentration values, using an even volume per sample @@ -1948,9 +1913,9 @@ def compute_shotgun_pooling_values_eqvol(sample_concs, total_vol=60.0): return sample_vols @staticmethod - def compute_shotgun_pooling_values_minvol( - sample_concs, sample_fracs=None, floor_vol=100, floor_conc=40, - total_nmol=.01): + def compute_pooling_values_minvol( + sample_concs, sample_fracs=None, floor_vol=2, floor_conc=16, + total=240, total_each=True, vol_constant=1): """Computes pooling volumes for samples based on concentration estimates of nM concentrations (`sample_concs`), taking a minimum volume of samples below a threshold. @@ -1966,7 +1931,7 @@ def compute_shotgun_pooling_values_minvol( pooling. Finally, total pooling size is determined by a target nanomolar - quantity (`total_nmol`, default .01). For a perfect 384 sample library, + quantity (`total`, default .01). For a perfect 384 sample library, in which you had all samples at a concentration of exactly 400 nM and wanted a total volume of 60 uL, this would be 0.024 nmol. @@ -1978,16 +1943,26 @@ def compute_shotgun_pooling_values_minvol( Parameters ---------- sample_concs: 2D array of float - nM sample concentrations + sample concentrations, with numerator same units as `total`. sample_fracs: 2D of float, optional fractional value for each sample (default 1/N) floor_vol: float, optional - volume (nL) at which samples below floor_conc will be pooled. + volume at which samples below floor_conc will be pooled. Default: 100 floor_conc: float, optional - minimum value (nM) for pooling at real estimated value. Default: 40 - total_nmol : float, optional - total number of nM to have in pool. Default: 0.01 + minimum value for pooling at real estimated value. Default: 40 + total : float, optional + total quantity (numerator) for pool. Unitless, but could represent + for example ng or nmol. Default: 240 + total_each : bool, optional + whether `total` refers to the quantity pooled *per sample* + (default; True) or to the total quantity of the pool. + vol_constant : float, optional + conversion factor between `sample_concs` demoninator and output + pooling volume units. E.g. if pooling ng/µL concentrations and + producing µL pool volumes, `vol_constant` = 1. If pooling nM + concentrations and producing nL pool volumes, `vol_constant` = + 10**-9. Default: 1 Returns ------- @@ -1995,73 +1970,19 @@ def compute_shotgun_pooling_values_minvol( the volumes in nL per each sample pooled """ if sample_fracs is None: - sample_fracs = np.ones(sample_concs.shape) / sample_concs.size + sample_fracs = np.ones(sample_concs.shape) + + if not total_each: + sample_fracs = sample_fracs / sample_concs.size # calculate volumetric fractions including floor val - sample_vols = (total_nmol * sample_fracs) / sample_concs - # convert L to nL - sample_vols *= 10**9 + sample_vols = (total * sample_fracs) / sample_concs + # convert volume from concentration units to pooling units + sample_vols *= vol_constant # drop volumes for samples below floor concentration to floor_vol sample_vols[sample_concs < floor_conc] = floor_vol return sample_vols - @staticmethod - def compute_shotgun_pooling_values_floor( - sample_concs, sample_fracs=None, min_conc=10, floor_conc=50, - total_nmol=.01): - """Computes pooling volumes for samples based on concentration - estimates of nM concentrations (`sample_concs`). - - Reads in concentration values in nM. Samples must be above a minimum - concentration threshold (`min_conc`, default 10 nM) to be included. - Samples above this threshold but below a given floor concentration - (`floor_conc`, default 50 nM) will be pooled as if they were at the - floor concentration, to avoid overdiluting the pool. - - Samples can be assigned a target molar fraction in the pool by passing - a np.array (`sample_fracs`, same shape as `sample_concs`) with - fractional values per sample. By default, will aim for equal molar - pooling. - - Finally, total pooling size is determined by a target nanomolar - quantity (`total_nmol`, default .01). For a perfect 384 sample library, - in which you had all samples at a concentration of exactly 400 nM and - wanted a total volume of 60 uL, this would be 0.024 nmol. - - Parameters - ---------- - sample_concs: 2D array of float - nM calculated by compute_qpcr_concentration - sample_fracs: 2D of float, optional - fractional value for each sample (default 1/N) - min_conc: float, optional - minimum nM concentration to be included in pool. Default: 10 - floor_conc: float, optional - minimum value for pooling for samples above min_conc. Default: 50 - total_nmol : float, optional - total number of nM to have in pool. Default 0.01 - - Returns - ------- - sample_vols: np.array of floats - the volumes in nL per each sample pooled - """ - if sample_fracs is None: - sample_fracs = np.ones(sample_concs.shape) / sample_concs.size - - # get samples above threshold - sample_fracs_pass = sample_fracs.copy() - sample_fracs_pass[sample_concs <= min_conc] = 0 - # renormalize to exclude lost samples - sample_fracs_pass *= 1/sample_fracs_pass.sum() - # floor concentration value - sample_concs_floor = sample_concs.copy() - sample_concs_floor[sample_concs < floor_conc] = floor_conc - # calculate volumetric fractions including floor val - sample_vols = (total_nmol * sample_fracs_pass) / sample_concs_floor - # convert L to nL - sample_vols *= 10**9 - return sample_vols @classmethod def create(cls, user, quantification_process, pool_name, volume, diff --git a/labman/db/tests/test_process.py b/labman/db/tests/test_process.py index 7cba20b1..79ce6783 100644 --- a/labman/db/tests/test_process.py +++ b/labman/db/tests/test_process.py @@ -628,11 +628,11 @@ def test_generate_echo_picklist(self): class TestQuantificationProcess(LabmanTestCase): - def test_compute_shotgun_pico_concentration(self): + def test_compute_pico_concentration(self): dna_vals = np.array([[10.14, 7.89, 7.9, 15.48], [7.86, 8.07, 8.16, 9.64], [12.29, 7.64, 7.32, 13.74]]) - obs = QuantificationProcess._compute_shotgun_pico_concentration( + obs = QuantificationProcess._compute_pico_concentration( dna_vals, size=400) exp = np.array([[38.4090909, 29.8863636, 29.9242424, 58.6363636], [29.7727273, 30.5681818, 30.9090909, 36.5151515], @@ -926,32 +926,34 @@ def test_generate_echo_picklist(self): class TestPoolingProcess(LabmanTestCase): - def test_compute_shotgun_pooling_values_eqvol(self): + def test_compute_pooling_values_eqvol(self): qpcr_conc = np.array( [[98.14626462, 487.8121413, 484.3480866, 2.183406934], [498.3536649, 429.0839787, 402.4270321, 140.1601735], [21.20533391, 582.9456031, 732.2655041, 7.545145988]]) - obs_sample_vols = PoolingProcess.compute_shotgun_pooling_values_eqvol( + obs_sample_vols = PoolingProcess.compute_pooling_values_eqvol( qpcr_conc, total_vol=60.0) exp_sample_vols = np.zeros([3, 4]) + 5000 npt.assert_allclose(obs_sample_vols, exp_sample_vols) - obs_sample_vols = PoolingProcess.compute_shotgun_pooling_values_eqvol( + obs_sample_vols = PoolingProcess.compute_pooling_values_eqvol( qpcr_conc, total_vol=60) npt.assert_allclose(obs_sample_vols, exp_sample_vols) - def test_compute_shotgun_pooling_values_minvol(self): + def test_compute_pooling_values_minvol(self): sample_concs = np.array([[1, 12, 400], [200, 40, 1]]) exp_vols = np.array([[100, 100, 4166.6666666666], [8333.33333333333, 41666.666666666, 100]]) - obs_vols = PoolingProcess.compute_shotgun_pooling_values_minvol( - sample_concs) + obs_vols = PoolingProcess.compute_pooling_values_minvol( + sample_concs, total=.01, floor_vol=100, floor_conc=40, + total_each=False, vol_constant=10**9) npt.assert_allclose(exp_vols, obs_vols) - def test_compute_shotgun_pooling_values_floor(self): - sample_concs = np.array([[1, 12, 400], [200, 40, 1]]) - exp_vols = np.array([[0, 50000, 6250], [12500, 50000, 0]]) - obs_vols = PoolingProcess.compute_shotgun_pooling_values_floor( + def test_compute_pooling_values_minvol_amplicon(self): + sample_concs = np.array([[1, 12, 40], [200, 40, 1]]) + exp_vols = np.array([[2, 2, 6], + [1.2, 6, 2]]) + obs_vols = PoolingProcess.compute_pooling_values_minvol( sample_concs) npt.assert_allclose(exp_vols, obs_vols) From a5f402ad2af27e09fd6a10841c6570e34dec6623 Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Tue, 20 Feb 2018 17:43:55 -0800 Subject: [PATCH 02/18] updated pooling process handler to consolidate amplicon and shotgun pooling functions --- labman/db/process.py | 4 +- .../process_handlers/pooling_process.py | 141 ++++++------------ 2 files changed, 46 insertions(+), 99 deletions(-) diff --git a/labman/db/process.py b/labman/db/process.py index 7048d687..def9f52d 100644 --- a/labman/db/process.py +++ b/labman/db/process.py @@ -1892,7 +1892,7 @@ def estimate_pool_conc_vol(sample_vols, sample_concs): return (pool_conc, total_vol) @staticmethod - def compute_pooling_values_eqvol(sample_concs, total_vol=60.0): + def compute_pooling_values_eqvol(sample_concs, total_vol=60.0, **kwargs): """Computes molar concentration of libraries from concentration values, using an even volume per sample @@ -1915,7 +1915,7 @@ def compute_pooling_values_eqvol(sample_concs, total_vol=60.0): @staticmethod def compute_pooling_values_minvol( sample_concs, sample_fracs=None, floor_vol=2, floor_conc=16, - total=240, total_each=True, vol_constant=1): + total=240, total_each=True, vol_constant=1, **kwargs): """Computes pooling volumes for samples based on concentration estimates of nM concentrations (`sample_concs`), taking a minimum volume of samples below a threshold. diff --git a/labman/gui/handlers/process_handlers/pooling_process.py b/labman/gui/handlers/process_handlers/pooling_process.py index 377a1857..6b66e63b 100644 --- a/labman/gui/handlers/process_handlers/pooling_process.py +++ b/labman/gui/handlers/process_handlers/pooling_process.py @@ -31,21 +31,7 @@ 'parameters': [('floor_vol', 'floor-vol-'), ('floor_conc', 'floor-conc-'), ('total_nmol', 'total-nm-'), - ('size', 'lib-size-')]}, - 'floor': {'function': PoolingProcess.compute_shotgun_pooling_values_floor, - 'parameters': [('floor_vol', 'floor-vol-'), - ('floor_conc', 'floor-conc-'), - ('total_nmol', 'total-nm-'), - ('size', 'lib-size-')]}, - # As everything, amplicon works differently here, we use this just for - # being able to retrieve the arguments - 'amplicon': {'function': None, - 'parameters': [('dna_amount', 'dna-amount-'), - ('min_val', 'min-val-'), - ('max_val', 'max-val-'), - ('blank_volume', 'blank-val-'), - ('robot', 'epmotion-'), - ('destination', 'dest-tube-')]}} + ('size', 'lib-size-')]}} HTML_POOL_PARAMS = { 'min': [{'prefix': 'floor-vol-', 'value': '100', @@ -59,40 +45,15 @@ 'min': '0.00001', 'step': '0.00001'}, {'prefix': 'lib-size-', 'value': '500', 'desc': 'Average library molecule size (bp):', 'min': '1', - 'step': '1'}], + 'step': '1'}, + {'prefix': 'epmotion-'}, {'prefix': 'dest-tube-'}], 'equal': [{'prefix': 'volume-', 'value': '200', 'desc': 'volume to pool per sample (nL):', 'min': '1', 'step': '1'}, {'prefix': 'lib-size-', 'value': '500', 'desc': 'Average library molecule size (bp):', 'min': '1', - 'step': '1'}], - 'floor': [{'prefix': 'floor-vol-', 'value': '10', - 'desc': 'Minimum concentration to be included in the ' - 'pool (nM):', - 'min': '1', 'step': '1'}, - {'prefix': 'floor-conc-', 'value': '50', - 'desc': 'Minimum value for pooling for samples above min ' - 'conc (nM):', - 'min': '1', 'step': '1'}, - {'prefix': 'total-nm-', 'value': '0.002', - 'desc': 'total number of nM to have in pool (nM):', - 'min': '0.00001', 'step': '0.00001'}, - {'prefix': 'lib-size-', 'value': '500', - 'desc': 'Average library molecule size (bp):', 'min': '1', - 'step': '1'}], - 'amplicon': [{'prefix': 'dna-amount-', 'value': '240', - 'desc': 'Total amount of DNA (ng):', 'min': '1', - 'step': '1'}, - {'prefix': 'min-val-', 'value': '1', - 'desc': 'Minimum concentration value (ng/μl):', - 'min': '0.001', 'step': '0.001'}, - {'prefix': 'max-val-', 'value': '15', - 'desc': 'Maximum concentration value (ng/μl):', - 'min': '0.001', 'step': '0.001'}, - {'prefix': 'blank-val-', 'value': '2', - 'desc': 'Blanks value (ng/μl):', 'min': '0.001', - 'step': '0.001'}, - {'prefix': 'epmotion-'}, {'prefix': 'dest-tube-'}]} + 'step': '1'}, + {'prefix': 'epmotion-'}, {'prefix': 'dest-tube-'}]} # quick function to create 2D representation of well-associated numbers @@ -166,67 +127,53 @@ class BasePoolHandler(BaseHandler): def _compute_pools(self, plate_info): plate_id = plate_info['plate-id'] func_name = plate_info['pool-func'] + plate_type = plate_info['plate-type'] + robot = plate_info['robot'] + dest = plate_info['destination'] func_info = POOL_FUNCS[func_name] function = func_info['function'] + plate = Plate(plate_id) quant_process = plate.quantification_process + # make params dictionary for function + params = {} + for arg, pfx in func_info['parameters']: + param_key = '%s%s' % (pfx, plate_id) + if param_key not in plate_info: + raise HTTPError( + 400, reason='Missing parameter %s' % param_key) + params[arg] = float(plate_info[param_key]) + + # compute molar concentrations + quant_process.compute_concentrations(size=params['size']) + + # calculate pooled values + raw_concs, comp_concs, comp_blanks, \ + plate_names = make_2D_arrays(plate, quant_process) + + if plate_type == 'amplicon': + pool_vals = function(comp_concs, **params) + if plate_type == 'shotgun': + params['total_each'] = False + params['vol_constant'] = 10**9 + pool_vals = function(comp_concs, **params) + + # TODO: adjust blank values if required + + # store output values output = {} - if func_name == 'amplicon': - params = {} - for arg, pfx in func_info['parameters']: - param_key = '%s%s' % (pfx, plate_id) - if param_key not in plate_info: - raise HTTPError( - 400, reason='Missing parameter %s' % param_key) - if arg in ('robot', 'destination'): - params[arg] = plate_info[param_key] - else: - params[arg] = float(plate_info[param_key]) - # Amplicon - output['robot'] = params.pop('robot') - output['destination'] = params.pop('destination') - output['func_data'] = {'function': 'amplicon', - 'parameters': params} - # Compute the normalized concentrations - quant_process.compute_concentrations(**params) - # Compute the pooling values - raw_concs, comp_concs, comp_blanks, \ - plate_names = make_2D_arrays(plate, quant_process) - output['raw_vals'] = raw_concs - output['comp_vals'] = comp_concs - output['pool_vals'] = comp_concs - output['pool_blanks'] = comp_blanks.tolist() - output['plate_names'] = plate_names.tolist() - else: - # Shotgun - params = {} - for arg, pfx in func_info['parameters']: - param_key = '%s%s' % (pfx, plate_id) - if param_key not in plate_info: - raise HTTPError( - 400, reason='Missing parameter %s' % param_key) - params[arg] = float(plate_info[param_key]) - # Compute the normalized concentrations - output['func_data'] = {'function': func_name, - 'parameters': deepcopy(params)} - size = params.pop('size') - quant_process.compute_concentrations(size=size) - # Compute the pooling values - raw_concs, comp_concs, comp_blanks, \ - plate_names = make_2D_arrays(plate, quant_process) - output['raw_vals'] = raw_concs - output['comp_vals'] = comp_concs - output['plate_names'] = plate_names.tolist() - output['pool_blanks'] = comp_blanks.tolist() - output['pool_vals'] = function(comp_concs, **params) - output['robot'] = None - output['destination'] = None - - # Make sure the results are JSON serializable + output['robot'] = robot + output['destination'] = dest + output['func_data'] = {'function': 'amplicon', + 'parameters': params} + output['raw_vals'] = raw_concs + output['comp_vals'] = comp_concs + output['pool_vals'] = pool_vals + output['pool_blanks'] = comp_blanks.tolist() output['plate_names'] = plate_names.tolist() output['plate_id'] = plate_id - output['pool_vals'] = output['pool_vals'] + return output From 47c7a04bac21ade81bcb675e0570a981c6b06d8b Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Wed, 21 Feb 2018 10:39:44 -0800 Subject: [PATCH 03/18] initial attempts to change HTML interaction --- .../process_handlers/pooling_process.py | 76 +++++++++++++------ .../test/test_pooling_process.py | 53 ++++++++----- labman/gui/templates/library_pooling.html | 76 +++++++------------ 3 files changed, 111 insertions(+), 94 deletions(-) diff --git a/labman/gui/handlers/process_handlers/pooling_process.py b/labman/gui/handlers/process_handlers/pooling_process.py index 6b66e63b..35d30905 100644 --- a/labman/gui/handlers/process_handlers/pooling_process.py +++ b/labman/gui/handlers/process_handlers/pooling_process.py @@ -24,36 +24,62 @@ POOL_FUNCS = { - 'equal': {'function': PoolingProcess.compute_shotgun_pooling_values_eqvol, + 'equal': {'function': PoolingProcess.compute_pooling_values_eqvol, 'parameters': [('total_vol', 'volume-'), - ('size', 'lib-size-')]}, - 'min': {'function': PoolingProcess.compute_shotgun_pooling_values_minvol, + ('size', 'lib-size-'), + ('robot', 'robot-'), + ('destination', 'dest-tube-')]}, + 'min': {'function': PoolingProcess.compute_pooling_values_minvol, 'parameters': [('floor_vol', 'floor-vol-'), ('floor_conc', 'floor-conc-'), - ('total_nmol', 'total-nm-'), - ('size', 'lib-size-')]}} + ('total', 'total-'), + ('size', 'lib-size-'), + ('robot', 'robot-'), + ('destination', 'dest-tube-')]}} -HTML_POOL_PARAMS = { +HTML_POOL_PARAMS_SHOTGUN = { 'min': [{'prefix': 'floor-vol-', 'value': '100', 'desc': 'volume for low conc samples (nL):', 'min': '1', 'step': '1'}, {'prefix': 'floor-conc-', 'value': '20', 'desc': 'minimum value for pooling at real estimated value (nM):', 'min': '0.1', 'step': '0.1'}, - {'prefix': 'total-nm-', 'value': '0.002', + {'prefix': 'total-', 'value': '0.002', 'desc': 'total number of nM to have in pool (nM):', 'min': '0.00001', 'step': '0.00001'}, {'prefix': 'lib-size-', 'value': '500', 'desc': 'Average library molecule size (bp):', 'min': '1', 'step': '1'}, - {'prefix': 'epmotion-'}, {'prefix': 'dest-tube-'}], + {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}], 'equal': [{'prefix': 'volume-', 'value': '200', 'desc': 'volume to pool per sample (nL):', 'min': '1', 'step': '1'}, {'prefix': 'lib-size-', 'value': '500', 'desc': 'Average library molecule size (bp):', 'min': '1', 'step': '1'}, - {'prefix': 'epmotion-'}, {'prefix': 'dest-tube-'}]} + {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}]} + +HTML_POOL_PARAMS_16S = { + 'min': [{'prefix': 'floor-vol-', 'value': '2', + 'desc': 'volume for low conc samples (µL):', 'min': '1', + 'step': '1'}, + {'prefix': 'floor-conc-', 'value': '16', + 'desc': 'minimum value for pooling at real estimated value (ng/µL):', + 'min': '0.1', 'step': '0.1'}, + {'prefix': 'total-', 'value': '240', + 'desc': 'total quantity of DNA to pool per sample (ng):', + 'min': '1', 'step': '0.1'}, + {'prefix': 'lib-size-', 'value': '500', + 'desc': 'Average library molecule size (bp):', 'min': '1', + 'step': '1'}, + {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}], + 'equal': [{'prefix': 'volume-', 'value': '5', + 'desc': 'volume to pool per sample (µL):', 'min': '1', + 'step': '1'}, + {'prefix': 'lib-size-', 'value': '500', + 'desc': 'Average library molecule size (bp):', 'min': '1', + 'step': '1'}, + {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}]} # quick function to create 2D representation of well-associated numbers @@ -128,8 +154,6 @@ def _compute_pools(self, plate_info): plate_id = plate_info['plate-id'] func_name = plate_info['pool-func'] plate_type = plate_info['plate-type'] - robot = plate_info['robot'] - dest = plate_info['destination'] func_info = POOL_FUNCS[func_name] function = func_info['function'] @@ -152,9 +176,11 @@ def _compute_pools(self, plate_info): raw_concs, comp_concs, comp_blanks, \ plate_names = make_2D_arrays(plate, quant_process) - if plate_type == 'amplicon': + if plate_type == '16S library prep': + params['total_each'] = True + params['vol_constant'] = 1 pool_vals = function(comp_concs, **params) - if plate_type == 'shotgun': + if plate_type == 'shotgun library prep': params['total_each'] = False params['vol_constant'] = 10**9 pool_vals = function(comp_concs, **params) @@ -163,9 +189,7 @@ def _compute_pools(self, plate_info): # store output values output = {} - output['robot'] = robot - output['destination'] = dest - output['func_data'] = {'function': 'amplicon', + output['func_data'] = {'function': func_name, 'parameters': params} output['raw_vals'] = raw_concs output['comp_vals'] = comp_concs @@ -249,13 +273,10 @@ def get(self): pool_blanks = pool_blanks.tolist() plate_names = plate_names.tolist() - if pool_func_data['function'] == 'amplicon': - pool_func_data['parameters']['epmotion-'] = process.robot.id - pool_func_data['parameters'][ - 'dest-tube-'] = process.destination elif len(plate_ids) > 0: content_types = {type(Plate(pid).get_well(1, 1).composition) for pid in plate_ids} + print(content_types) if len(content_types) > 1: raise HTTPError(400, reason='Plates contain different types ' 'of compositions') @@ -263,9 +284,16 @@ def get(self): if content_types.pop() == LibraryPrep16SComposition else 'shotgun library prep') - epmotions = Equipment.list_equipment('EpMotion') + robots = Equipment.list_equipment('EpMotion') + \ + Equipment.list_equipment('echo') + + if plate_type == '16S library prep': + HTML_POOL_PARAMS = HTML_POOL_PARAMS_16S + else: + HTML_POOL_PARAMS = HTML_POOL_PARAMS_SHOTGUN + self.render('library_pooling.html', plate_ids=plate_ids, - epmotions=epmotions, pool_params=HTML_POOL_PARAMS, + robots=robots, pool_params=HTML_POOL_PARAMS, input_plate=input_plate, pool_func_data=pool_func_data, process_id=process_id, pool_values=pool_values, plate_type=plate_type, pool_blanks=pool_blanks, @@ -274,9 +302,9 @@ def get(self): @authenticated def post(self): plates_info = json_decode(self.get_argument('plates-info')) - results = [] for pinfo in plates_info: + plate_result = self._compute_pools(pinfo) plate = Plate(plate_result['plate_id']) pool_name = 'Pool from plate %s (%s)' % ( @@ -320,8 +348,6 @@ def post(self): # We don't need to return these values to the interface output.pop('raw_vals') output.pop('comp_vals') - output.pop('robot') - output.pop('destination') output.pop('func_data') self.write(output) diff --git a/labman/gui/handlers/process_handlers/test/test_pooling_process.py b/labman/gui/handlers/process_handlers/test/test_pooling_process.py index 39a32b46..3dd38b0e 100644 --- a/labman/gui/handlers/process_handlers/test/test_pooling_process.py +++ b/labman/gui/handlers/process_handlers/test/test_pooling_process.py @@ -12,14 +12,20 @@ from labman.gui.testing import TestHandlerBase from labman.gui.handlers.process_handlers.pooling_process import ( - POOL_FUNCS, HTML_POOL_PARAMS) + POOL_FUNCS, HTML_POOL_PARAMS_16S, HTML_POOL_PARAMS_SHOTGUN) class TestPoolingProcessHandlers(TestHandlerBase): - def test_html_backend_pairing(self): + def test_html_backend_pairing_16S(self): for key, vals in POOL_FUNCS.items(): pyparams = [html_prefix for _, html_prefix in vals['parameters']] - htmlpfx = [v['prefix'] for v in HTML_POOL_PARAMS[key]] + htmlpfx = [v['prefix'] for v in HTML_POOL_PARAMS_16S[key]] + self.assertCountEqual(pyparams, htmlpfx) + + def test_html_backend_pairing_shotgun(self): + for key, vals in POOL_FUNCS.items(): + pyparams = [html_prefix for _, html_prefix in vals['parameters']] + htmlpfx = [v['prefix'] for v in HTML_POOL_PARAMS_SHOTGUN[key]] self.assertCountEqual(pyparams, htmlpfx) def test_get_pool_pool_process_handler(self): @@ -79,9 +85,10 @@ def test_get_library_pool_process_handler(self): def test_post_library_pool_process_handler(self): # Amplicon test data = {'plates-info': json_encode([{ - 'plate-id': 23, 'pool-func': 'amplicon', 'dna-amount-23': 240, - 'min-val-23': 1, 'max-val-23': 15, 'blank-val-23': 2, - 'epmotion-23': 10, 'dest-tube-23': 1}])} + 'plate-id': 23, 'pool-func': 'min', + 'plate-type': '16S library prep', + 'total-23': 240, 'floor-vol-23': 2, 'floor-conc-23': 16, + 'robot-23': 10, 'dest-tube-23': 1}])} response = self.post('/process/poollibraries', data) self.assertEqual(response.code, 200) obs = json_decode(response.body) @@ -90,33 +97,38 @@ def test_post_library_pool_process_handler(self): # Shotgun test data = {'plates-info': json_encode([{ - 'plate-id': 26, 'pool-func': 'equal', 'volume-26': 200, - 'lib-size-26': 500}])} + 'plate-id': 26, 'pool-func': 'equal', + 'plate-type': 'shotgun library prep', 'volume-26': 200, + 'lib-size-26': 500, 'robot-23': 10, 'dest-tube-23': 1}])} response = self.post('/process/poollibraries', data) self.assertEqual(response.code, 200) obs = json_decode(response.body) self.assertEqual(len(obs), 1) self.assertCountEqual(obs[0], ['plate-id', 'process-id']) - # Failure amplicon + # Failure amplicon: missing dest-tube- data = {'plates-info': json_encode([{ - 'plate-id': 23, 'pool-func': 'amplicon', 'dna-amount-23': 240, - 'min-val-23': 1, 'max-val-23': 15, 'blank-val-23': 2, - 'epmotion-23': 10}])} + 'plate-id': 23, 'pool-func': 'min', + 'plate-type': '16S library prep', + 'total-23': 240, 'floor-vol-23': 2, 'floor-conc-23': 16, + 'robot-23': 10}])} response = self.post('/process/poollibraries', data) self.assertEqual(response.code, 400) - # Failure shotgun + # Failure shotgun: missing lib-size- data = {'plates-info': json_encode([{ - 'plate-id': 26, 'pool-func': 'equal', 'volume-26': 200}])} + 'plate-id': 26, 'pool-func': 'equal', + 'plate-type': 'shotgun library prep', + 'robot-23': 10, 'dest-tube-23': 1, 'volume-26': 200}])} response = self.post('/process/poollibraries', data) self.assertEqual(response.code, 400) def test_post_compute_library_pool_values_handler(self): data = {'plate-info': json_encode({ - 'plate-id': 23, 'pool-func': 'amplicon', 'dna-amount-23': 240, - 'min-val-23': 1, 'max-val-23': 15, 'blank-val-23': 2, - 'epmotion-23': 10, 'dest-tube-23': 1})} + 'plate-id': 23, 'pool-func': 'min', + 'plate-type': '16S library prep', + 'total-23': 240, 'floor-vol-23': 2, 'floor-conc-23': 16, + 'robot-23': 10, 'dest-tube-23': 1})} response = self.post('/process/compute_pool', data) self.assertEqual(response.code, 200) self.assertCountEqual(json_decode(response.body), @@ -124,9 +136,10 @@ def test_post_compute_library_pool_values_handler(self): 'plate_names']) data = {'plate-info': json_encode({ - 'plate-id': 23, 'pool-func': 'amplicon', 'dna-amount-23': 240, - 'min-val-23': 1, 'max-val-23': 15, 'blank-val-23': 2, - 'epmotion-23': 10})} + 'plate-id': 23, 'pool-func': 'min', + 'plate-type': '16S library prep', + 'total-23': 240, 'floor-vol-23': 2, 'floor-conc-23': 16, + 'robot-23': 10})} response = self.post('/process/compute_pool', data) self.assertEqual(response.code, 400) diff --git a/labman/gui/templates/library_pooling.html b/labman/gui/templates/library_pooling.html index ca90ae33..dce5c49f 100644 --- a/labman/gui/templates/library_pooling.html +++ b/labman/gui/templates/library_pooling.html @@ -21,7 +21,7 @@ disableAll(); bootstrapAlert('Loading data...', 'info'); var funcData = {% raw pool_func_data %}; - var plateType = funcData['function'] === 'amplicon' ? '16S library prep' : 'shotgun library prep'; + var plateType = {% raw plate_type %}; var defaultClipping = clippingForPlateType(plateType); $('#plate-type-select').val(plateType); @@ -91,13 +91,9 @@ var plateType = $('#plate-type-select').val(); var parameterName, poolFunc; - if (plateType === '16S library prep') { - poolFunc = 'amplicon'; - } else { - poolFunc = $(elem).find('select').val(); - } + poolFunc = $(elem).find('select').val(); - var plateInfo = {'plate-id': plateId, 'pool-func': poolFunc}; + var plateInfo = {'plate-id': plateId, 'pool-func': poolFunc, 'plate-type': plateType}; $.each(poolParams[poolFunc], function(idx, paramElem) { parameterName = paramElem['prefix'] + plateId; plateInfo[parameterName] = $('#' + parameterName).val(); @@ -172,31 +168,20 @@ var isPoolBtnDisabled = $('#compute-pool-btn').prop('disabled'); var disabled = !isPoolBtnDisabled; var plateType = $('#plate-type-select').val(); - if (plateType === '16S library prep') { - $.each(plates, function(idx, elem) { - var plateId = $(elem).attr('pm-data-plate-id'); - $.each(poolParams['amplicon'], function(idx, paramElem) { + $.each(plates, function(idx, elem) { + var plateId = $(elem).attr('pm-data-plate-id'); + var poolFunc = $(elem).find('select').val(); + if (poolFunc === null) { + $('#compute-pool-btn').prop('disabled', true); + disabled = true; + } else { + $.each(poolParams[poolFunc], function(idx, paramElem) { var isEmpty = $('#' + paramElem['prefix'] + plateId).val() === ''; $('#compute-pool-btn').prop('disabled', isEmpty || isPoolBtnDisabled); disabled = disabled || isEmpty; }); - }); - } else { - $.each(plates, function(idx, elem) { - var plateId = $(elem).attr('pm-data-plate-id'); - var poolFunc = $(elem).find('select').val(); - if (poolFunc === null) { - $('#compute-pool-btn').prop('disabled', true); - disabled = true; - } else { - $.each(poolParams[poolFunc], function(idx, paramElem) { - var isEmpty = $('#' + paramElem['prefix'] + plateId).val() === ''; - $('#compute-pool-btn').prop('disabled', isEmpty || isPoolBtnDisabled); - disabled = disabled || isEmpty; - }); - } - }); - } + } + }); $('#pool-btn').prop('disabled', disabled); } }; @@ -220,7 +205,13 @@ var $target = $('#pool-params-div-' + plateId); $target.empty(); $.each(poolParams[$(this).val()], function(idx, elem) { - createNumberInputDOM($('#pool-params-div-' + plateId), plateId, paramOnChangeCallback, elem['desc'], elem['value'], elem['prefix'], elem['step'], elem['min']); + if (elem['prefix'] === 'robot-') { + createSelectDOM($('#pool-params-div-' + plateId), plateId, paramOnChangeCallback, 'Pooling robot', {% raw robots %}, elem['prefix'], 'Select robot...'); + } else if (elem['prefix'] === 'dest-tube-') { + createTextInputDOM($('#pool-params-div-' + plateId), plateId, paramOnChangeCallback, 'Destination tube', '1', elem['prefix']); + } else { + createNumberInputDOM($('#pool-params-div-' + plateId), plateId, paramOnChangeCallback, elem['desc'], elem['value'], elem['prefix'], elem['step'], elem['min']); + } }); enableComputePoolValues(); poolingChecks(); @@ -259,26 +250,13 @@ $divElem.append(''); var $formDiv = $("
").addClass('form-horizontal').appendTo($divElem); var plateType = $('#plate-type-select').val(); - if (plateType === '16S library prep') { - // Amplicon library There is only 1 library prep function - add the parameters - $.each(poolParams['amplicon'], function(idx, elem) { - if (elem['prefix'] === 'epmotion-') { - createSelectDOM($formDiv, plateId, paramOnChangeCallback, 'EpMotion robot', {% raw epmotions %}, elem['prefix'], 'Select EpMotion...'); - } else if (elem['prefix'] === 'dest-tube-') { - createTextInputDOM($formDiv, plateId, paramOnChangeCallback, 'EpMotion destination tube', '1', elem['prefix']); - } else { - createNumberInputDOM($formDiv, plateId, paramOnChangeCallback, elem['desc'], elem['value'], elem['prefix'], elem['step'], elem['min']); - } - }); - enableComputePoolValues(); - } else { - // Add a select for choosing the pooling function - var poolFuncOpts = [{'func_id': 'equal', 'external_id': 'Equal volume'}, - {'func_id': 'min', 'external_id': 'Minimum volume'}, - {'func_id': 'floor', 'external_id': 'Floor volume'}]; - createSelectDOM($formDiv, plateId, poolFuncCallback, 'Pooling function', poolFuncOpts, 'pool-func-', 'Choose pooling function...', 'func_id'); - $('
').addClass('form-horizontal').attr('id', 'pool-params-div-' + plateId).appendTo($formDiv); - } + + // Add a select for choosing the pooling function + var poolFuncOpts = [{'func_id': 'equal', 'external_id': 'Equal volume'}, + {'func_id': 'min', 'external_id': 'Minimum volume'}]; + createSelectDOM($formDiv, plateId, poolFuncCallback, 'Pooling function', poolFuncOpts, 'pool-func-', 'Choose pooling function...', 'func_id'); + $('
').addClass('form-horizontal').attr('id', 'pool-params-div-' + plateId).appendTo($formDiv); + // Add a space for the pooling function results $('
').addClass('form-group').appendTo($formDiv).attr('id', 'pool-results-' + plateId); // Add the element to the plate list From 43d3c16be078be781e8b087531e32ee05b077642 Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Wed, 21 Feb 2018 14:47:52 -0800 Subject: [PATCH 04/18] updated test db to include new pooling function signatures --- labman/db/support_files/populate_test_db.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/labman/db/support_files/populate_test_db.sql b/labman/db/support_files/populate_test_db.sql index 93889123..0dd4f3a5 100644 --- a/labman/db/support_files/populate_test_db.sql +++ b/labman/db/support_files/populate_test_db.sql @@ -569,9 +569,10 @@ BEGIN RETURNING process_id INTO p_pool_process_id; INSERT INTO qiita.pooling_process (process_id, quantification_process_id, robot_id, destination, pooling_function_data) - VALUES (p_pool_process_id, pg_quant_subprocess_id, proc_robot_id, 1, '{"function": "amplicon", "parameters": {"dna-amount-": 240, "min-val-": 1, "max-val-": 15, "blank-val-": 2}}'::json) + VALUES (p_pool_process_id, pg_quant_subprocess_id, proc_robot_id, 1, '{"function": "amplicon", "parameters": {"total-": 240, "floor-vol-": 2, "floor-conc-": 16}}'::json) RETURNING pooling_process_id INTO p_pool_subprocess_id; + ---------------------------------------- ------ SEQUENCING POOLING PROCESS ------ ---------------------------------------- From 57787639b47141d73e8a3f60acbf8b44ddc292ed Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Wed, 21 Feb 2018 14:48:27 -0800 Subject: [PATCH 05/18] passing tests --- .../process_handlers/pooling_process.py | 11 +++++---- .../test/test_pooling_process.py | 24 ++++++++++--------- labman/gui/templates/library_pooling.html | 9 +++---- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/labman/gui/handlers/process_handlers/pooling_process.py b/labman/gui/handlers/process_handlers/pooling_process.py index 35d30905..83d7b537 100644 --- a/labman/gui/handlers/process_handlers/pooling_process.py +++ b/labman/gui/handlers/process_handlers/pooling_process.py @@ -81,6 +81,8 @@ 'step': '1'}, {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}]} +HTML_POOL_PARAMS = {'16S library prep': HTML_POOL_PARAMS_16S, + 'shotgun library prep': HTML_POOL_PARAMS_SHOTGUN} # quick function to create 2D representation of well-associated numbers def make_2D_arrays(plate, quant_process): @@ -157,6 +159,8 @@ def _compute_pools(self, plate_info): func_info = POOL_FUNCS[func_name] function = func_info['function'] + print('foo') + plate = Plate(plate_id) quant_process = plate.quantification_process @@ -197,6 +201,8 @@ def _compute_pools(self, plate_info): output['pool_blanks'] = comp_blanks.tolist() output['plate_names'] = plate_names.tolist() output['plate_id'] = plate_id + output['destination'] = params['destination'] + output['robot'] = params['robot'] return output @@ -287,11 +293,6 @@ def get(self): robots = Equipment.list_equipment('EpMotion') + \ Equipment.list_equipment('echo') - if plate_type == '16S library prep': - HTML_POOL_PARAMS = HTML_POOL_PARAMS_16S - else: - HTML_POOL_PARAMS = HTML_POOL_PARAMS_SHOTGUN - self.render('library_pooling.html', plate_ids=plate_ids, robots=robots, pool_params=HTML_POOL_PARAMS, input_plate=input_plate, pool_func_data=pool_func_data, diff --git a/labman/gui/handlers/process_handlers/test/test_pooling_process.py b/labman/gui/handlers/process_handlers/test/test_pooling_process.py index 3dd38b0e..8fec08a4 100644 --- a/labman/gui/handlers/process_handlers/test/test_pooling_process.py +++ b/labman/gui/handlers/process_handlers/test/test_pooling_process.py @@ -83,23 +83,25 @@ def test_get_library_pool_process_handler(self): self.assertEqual(response.code, 404) def test_post_library_pool_process_handler(self): - # Amplicon test + + # Shotgun test data = {'plates-info': json_encode([{ - 'plate-id': 23, 'pool-func': 'min', - 'plate-type': '16S library prep', - 'total-23': 240, 'floor-vol-23': 2, 'floor-conc-23': 16, - 'robot-23': 10, 'dest-tube-23': 1}])} + 'plate-id': 26, 'pool-func': 'equal', + 'plate-type': 'shotgun library prep', 'volume-26': 200, + 'lib-size-26': 500, 'robot-26': 10, 'dest-tube-26': 1}])} + response = self.post('/process/poollibraries', data) self.assertEqual(response.code, 200) obs = json_decode(response.body) self.assertEqual(len(obs), 1) self.assertCountEqual(obs[0], ['plate-id', 'process-id']) - # Shotgun test + # Amplicon test data = {'plates-info': json_encode([{ - 'plate-id': 26, 'pool-func': 'equal', - 'plate-type': 'shotgun library prep', 'volume-26': 200, - 'lib-size-26': 500, 'robot-23': 10, 'dest-tube-23': 1}])} + 'plate-id': 23, 'pool-func': 'min', + 'plate-type': '16S library prep', + 'total-23': 240, 'floor-vol-23': 2, 'floor-conc-23': 16, + 'lib-size-23': 500, 'robot-23': 10, 'dest-tube-23': 1}])} response = self.post('/process/poollibraries', data) self.assertEqual(response.code, 200) obs = json_decode(response.body) @@ -128,12 +130,12 @@ def test_post_compute_library_pool_values_handler(self): 'plate-id': 23, 'pool-func': 'min', 'plate-type': '16S library prep', 'total-23': 240, 'floor-vol-23': 2, 'floor-conc-23': 16, - 'robot-23': 10, 'dest-tube-23': 1})} + 'lib-size-23': 500, 'robot-23': 10, 'dest-tube-23': 1})} response = self.post('/process/compute_pool', data) self.assertEqual(response.code, 200) self.assertCountEqual(json_decode(response.body), ['plate_id', 'pool_vals', 'pool_blanks', - 'plate_names']) + 'plate_names', 'destination', 'robot']) data = {'plate-info': json_encode({ 'plate-id': 23, 'pool-func': 'min', diff --git a/labman/gui/templates/library_pooling.html b/labman/gui/templates/library_pooling.html index dce5c49f..c1516b8f 100644 --- a/labman/gui/templates/library_pooling.html +++ b/labman/gui/templates/library_pooling.html @@ -34,7 +34,7 @@ $('#pool-func-' + plateId).val(funcData['function']); // Build the function FROM $('#pool-func-' + plateId).trigger($.Event('change')); - $.each(poolParams[funcData['function']], function(idx, paramElem) { + $.each(poolParams[plateType][funcData['function']], function(idx, paramElem) { parameterName = paramElem['prefix'] + plateId; $('#' + parameterName).val(funcData['parameters'][paramElem['prefix']]); @@ -94,7 +94,7 @@ poolFunc = $(elem).find('select').val(); var plateInfo = {'plate-id': plateId, 'pool-func': poolFunc, 'plate-type': plateType}; - $.each(poolParams[poolFunc], function(idx, paramElem) { + $.each(poolParams[plateType][poolFunc], function(idx, paramElem) { parameterName = paramElem['prefix'] + plateId; plateInfo[parameterName] = $('#' + parameterName).val(); }); @@ -175,7 +175,7 @@ $('#compute-pool-btn').prop('disabled', true); disabled = true; } else { - $.each(poolParams[poolFunc], function(idx, paramElem) { + $.each(poolParams[plateType][poolFunc], function(idx, paramElem) { var isEmpty = $('#' + paramElem['prefix'] + plateId).val() === ''; $('#compute-pool-btn').prop('disabled', isEmpty || isPoolBtnDisabled); disabled = disabled || isEmpty; @@ -203,8 +203,9 @@ function poolFuncCallback() { var plateId = $(this).attr('plate-id'); var $target = $('#pool-params-div-' + plateId); + var plateType = $('#plate-type-select').val(); $target.empty(); - $.each(poolParams[$(this).val()], function(idx, elem) { + $.each(poolParams[plateType][$(this).val()], function(idx, elem) { if (elem['prefix'] === 'robot-') { createSelectDOM($('#pool-params-div-' + plateId), plateId, paramOnChangeCallback, 'Pooling robot', {% raw robots %}, elem['prefix'], 'Select robot...'); } else if (elem['prefix'] === 'dest-tube-') { From d0855cc0af88db75e794e866f19e455149621581 Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Wed, 21 Feb 2018 15:04:18 -0800 Subject: [PATCH 06/18] updated test db with more realistic values for quantified amplicon library plate --- labman/db/support_files/populate_test_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labman/db/support_files/populate_test_db.sql b/labman/db/support_files/populate_test_db.sql index 0dd4f3a5..483e4f6d 100644 --- a/labman/db/support_files/populate_test_db.sql +++ b/labman/db/support_files/populate_test_db.sql @@ -659,7 +659,7 @@ BEGIN -- Quantify plate pools INSERT INTO qiita.concentration_calculation (quantitated_composition_id, upstream_process_id, raw_concentration) - VALUES (p_pool_composition_id, ppg_quant_subprocess_id, 1.5); + VALUES (p_pool_composition_id, ppg_quant_subprocess_id, 25); -- Pool sequencing run INSERT INTO qiita.container (container_type_id, latest_upstream_process_id, remaining_volume) From bbf5a96096acf14c7e76cc8553600a21ba410a89 Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Wed, 21 Feb 2018 15:04:56 -0800 Subject: [PATCH 07/18] fixed to do amplicon pooling by ng/uL and not molarity --- labman/gui/handlers/process_handlers/pooling_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labman/gui/handlers/process_handlers/pooling_process.py b/labman/gui/handlers/process_handlers/pooling_process.py index 83d7b537..b65e6293 100644 --- a/labman/gui/handlers/process_handlers/pooling_process.py +++ b/labman/gui/handlers/process_handlers/pooling_process.py @@ -183,7 +183,7 @@ def _compute_pools(self, plate_info): if plate_type == '16S library prep': params['total_each'] = True params['vol_constant'] = 1 - pool_vals = function(comp_concs, **params) + pool_vals = function(raw_concs, **params) if plate_type == 'shotgun library prep': params['total_each'] = False params['vol_constant'] = 10**9 From 93ec344973fa6d4829d0b0a65d7a7989702081b2 Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Thu, 19 Apr 2018 09:13:45 -0700 Subject: [PATCH 08/18] updating tests and default data to be more sensible for amplicon pooling --- labman/db/support_files/populate_test_db.sql | 2 +- labman/db/tests/test_composition.py | 2 +- labman/db/tests/test_process.py | 25 +++++++++++-------- .../process_handlers/pooling_process.py | 4 +-- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/labman/db/support_files/populate_test_db.sql b/labman/db/support_files/populate_test_db.sql index 483e4f6d..95ad7d3f 100644 --- a/labman/db/support_files/populate_test_db.sql +++ b/labman/db/support_files/populate_test_db.sql @@ -952,7 +952,7 @@ BEGIN -- Quantification INSERT INTO qiita.concentration_calculation (quantitated_composition_id, upstream_process_id, raw_concentration, computed_concentration) - VALUES (lib_prep_16s_composition_id, pg_quant_subprocess_id, 1.5, 1.5); + VALUES (lib_prep_16s_composition_id, pg_quant_subprocess_id, 20., 60.6060); -- Pool plate INSERT INTO qiita.pool_composition_components (output_pool_composition_id, input_composition_id, input_volume, percentage_of_output) diff --git a/labman/db/tests/test_composition.py b/labman/db/tests/test_composition.py index 8a922b03..770f76c1 100644 --- a/labman/db/tests/test_composition.py +++ b/labman/db/tests/test_composition.py @@ -312,7 +312,7 @@ def test_pool_composition_attributes(self): exp = {'composition': LibraryPrep16SComposition(1), 'input_volume': 1.0, 'percentage_of_output': 0} self.assertEqual(obs_comp[0], exp) - self.assertEqual(obs.raw_concentration, 1.5) + self.assertEqual(obs.raw_concentration, 25.0) def test_primer_set_attributes(self): obs = PrimerSet(1) diff --git a/labman/db/tests/test_process.py b/labman/db/tests/test_process.py index 79ce6783..6d76a14a 100644 --- a/labman/db/tests/test_process.py +++ b/labman/db/tests/test_process.py @@ -766,10 +766,15 @@ def test_create(self): user = User('test@foo.bar') plate = Plate(23) concentrations = np.around(np.random.rand(8, 12), 6) - # Add some known values + + # Add some known values for DNA concentration concentrations[0][0] = 3 concentrations[0][1] = 4 concentrations[0][2] = 40 + # Set blank wells to zero DNA concentrations + concentrations[7] = np.zeros_like(concentrations[7]) + + # add DNA concentrations to plate and check for sanity obs = QuantificationProcess.create(user, plate, concentrations) self.assertEqual(obs.date, date.today()) self.assertEqual(obs.personnel, user) @@ -781,19 +786,17 @@ def test_create(self): self.assertEqual(obs_c[12][0], LibraryPrep16SComposition(13)) npt.assert_almost_equal(obs_c[12][1], concentrations[1][0]) self.assertIsNone(obs_c[12][2]) + + # compute library concentrations (nM) from DNA concentrations (ng/uL) obs.compute_concentrations() obs_c = obs.concentrations - # The values that we know - npt.assert_almost_equal(obs_c[0][2], 80) - npt.assert_almost_equal(obs_c[1][2], 60) - npt.assert_almost_equal(obs_c[2][2], 0) - # The rest (except last row) are 1 because np.random - # generates numbers < 1 - for i in range(3, 84): - npt.assert_almost_equal(obs_c[i][2], 1) - # Last row are all 2 because they're blanks + # Check the values that we know + npt.assert_almost_equal(obs_c[0][2], 9.09091) + npt.assert_almost_equal(obs_c[1][2], 12.1212) + npt.assert_almost_equal(obs_c[2][2], 121.212) + # Last row are all 0 because they're blanks for i in range(84, 95): - npt.assert_almost_equal(obs_c[i][2], 2) + npt.assert_almost_equal(obs_c[i][2], 0) concentrations = np.around(np.random.rand(16, 24), 6) # Add some known values diff --git a/labman/gui/handlers/process_handlers/pooling_process.py b/labman/gui/handlers/process_handlers/pooling_process.py index b65e6293..3b44986d 100644 --- a/labman/gui/handlers/process_handlers/pooling_process.py +++ b/labman/gui/handlers/process_handlers/pooling_process.py @@ -159,8 +159,6 @@ def _compute_pools(self, plate_info): func_info = POOL_FUNCS[func_name] function = func_info['function'] - print('foo') - plate = Plate(plate_id) quant_process = plate.quantification_process @@ -282,7 +280,7 @@ def get(self): elif len(plate_ids) > 0: content_types = {type(Plate(pid).get_well(1, 1).composition) for pid in plate_ids} - print(content_types) + if len(content_types) > 1: raise HTTPError(400, reason='Plates contain different types ' 'of compositions') From b88c06ed599376b3d1c55fe95a29e259c6357f28 Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Wed, 25 Apr 2018 13:37:39 -0700 Subject: [PATCH 09/18] updated test db to have lower amplicon blank concentrations --- labman/db/support_files/populate_test_db.sql | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/labman/db/support_files/populate_test_db.sql b/labman/db/support_files/populate_test_db.sql index 95ad7d3f..05b13983 100644 --- a/labman/db/support_files/populate_test_db.sql +++ b/labman/db/support_files/populate_test_db.sql @@ -951,8 +951,13 @@ BEGIN VALUES (lib_prep_16s_composition_id, gdna_subcomposition_id, primer_comp_id); -- Quantification - INSERT INTO qiita.concentration_calculation (quantitated_composition_id, upstream_process_id, raw_concentration, computed_concentration) - VALUES (lib_prep_16s_composition_id, pg_quant_subprocess_id, 20., 60.6060); + IF idx_row_well <= 7 THEN + INSERT INTO qiita.concentration_calculation (quantitated_composition_id, upstream_process_id, raw_concentration, computed_concentration) + VALUES (lib_prep_16s_composition_id, pg_quant_subprocess_id, 20., 60.6060); + ELSE + INSERT INTO qiita.concentration_calculation (quantitated_composition_id, upstream_process_id, raw_concentration, computed_concentration) + VALUES (lib_prep_16s_composition_id, pg_quant_subprocess_id, 1., 3.0303); + END IF; -- Pool plate INSERT INTO qiita.pool_composition_components (output_pool_composition_id, input_composition_id, input_volume, percentage_of_output) From d5bb161aadb72d5cb019c7903e931bba8fc4ddb7 Mon Sep 17 00:00:00 2001 From: Jon Sanders Date: Wed, 25 Apr 2018 16:44:25 -0700 Subject: [PATCH 10/18] updated pooling interface to expose blank options --- .../process_handlers/pooling_process.py | 32 ++++++++++++-- labman/gui/static/js/labman.js | 42 ++++++++++++++----- labman/gui/templates/library_pooling.html | 7 ++++ 3 files changed, 67 insertions(+), 14 deletions(-) diff --git a/labman/gui/handlers/process_handlers/pooling_process.py b/labman/gui/handlers/process_handlers/pooling_process.py index 3b44986d..53d8bb26 100644 --- a/labman/gui/handlers/process_handlers/pooling_process.py +++ b/labman/gui/handlers/process_handlers/pooling_process.py @@ -50,14 +50,26 @@ {'prefix': 'lib-size-', 'value': '500', 'desc': 'Average library molecule size (bp):', 'min': '1', 'step': '1'}, - {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}], + {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}, + {'prefix': 'blank-number-', 'value': '', + 'desc': 'Pool only highest N blanks, N=', 'min': 0, + 'step': 1}, + {'prefix': 'blank-vol-', 'value': '', + 'desc': 'Pool all blanks at volume (nL):', 'min': 0, + 'step': 2.5}], 'equal': [{'prefix': 'volume-', 'value': '200', 'desc': 'volume to pool per sample (nL):', 'min': '1', 'step': '1'}, {'prefix': 'lib-size-', 'value': '500', 'desc': 'Average library molecule size (bp):', 'min': '1', 'step': '1'}, - {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}]} + {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}, + {'prefix': 'blank-number-', 'value': '', + 'desc': 'Pool only highest N blanks, N=', 'min': 0, + 'step': 1}, + {'prefix': 'blank-vol-', 'value': '', + 'desc': 'Pool all blanks at volume (nL):', 'min': 0, + 'step': 2.5}]} HTML_POOL_PARAMS_16S = { 'min': [{'prefix': 'floor-vol-', 'value': '2', @@ -72,14 +84,26 @@ {'prefix': 'lib-size-', 'value': '500', 'desc': 'Average library molecule size (bp):', 'min': '1', 'step': '1'}, - {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}], + {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}, + {'prefix': 'blank-number-', 'value': 2, + 'desc': 'Pool only highest N blanks, N=', 'min': 0, + 'step': 1}, + {'prefix': 'blank-vol-', 'value': 5, + 'desc': 'Pool all blanks at volume (µL):', 'min': 0, + 'step': 0.1}], 'equal': [{'prefix': 'volume-', 'value': '5', 'desc': 'volume to pool per sample (µL):', 'min': '1', 'step': '1'}, {'prefix': 'lib-size-', 'value': '500', 'desc': 'Average library molecule size (bp):', 'min': '1', 'step': '1'}, - {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}]} + {'prefix': 'robot-'}, {'prefix': 'dest-tube-'}, + {'prefix': 'blank-number-', 'value': 2, + 'desc': 'Pool only highest N blanks, N=', 'min': 0, + 'step': 1}, + {'prefix': 'blank-vol-', 'value': 5, + 'desc': 'Pool all blanks at volume (µL):', 'min': 0, + 'step': 0.1}]} HTML_POOL_PARAMS = {'16S library prep': HTML_POOL_PARAMS_16S, 'shotgun library prep': HTML_POOL_PARAMS_SHOTGUN} diff --git a/labman/gui/static/js/labman.js b/labman/gui/static/js/labman.js index 64a3fac5..f1e300df 100644 --- a/labman/gui/static/js/labman.js +++ b/labman/gui/static/js/labman.js @@ -21,8 +21,8 @@ function disableAll() { **/ function createPlateNameInputDOM($targetDiv, plateId, checksCallback, label, defaultValue) { var $rowDiv = $('
').addClass('form-group').appendTo($targetDiv); - $('