diff --git a/microsetta_private_api/admin/admin_impl.py b/microsetta_private_api/admin/admin_impl.py index eefbe416b..b52975852 100644 --- a/microsetta_private_api/admin/admin_impl.py +++ b/microsetta_private_api/admin/admin_impl.py @@ -153,6 +153,16 @@ def scan_barcode(token_info, sample_barcode, body): return response +def get_observations(token_info, sample_barcode): + validate_admin_access(token_info) + + with Transaction() as t: + admin_repo = AdminRepo(t) + observations = admin_repo.\ + retrieve_observations_by_project(sample_barcode) + return jsonify(observations), 200 + + def sample_pulldown_single_survey(token_info, sample_barcode, survey_template_id): diff --git a/microsetta_private_api/admin/tests/test_admin_api.py b/microsetta_private_api/admin/tests/test_admin_api.py index c7d8df302..76db43497 100644 --- a/microsetta_private_api/admin/tests/test_admin_api.py +++ b/microsetta_private_api/admin/tests/test_admin_api.py @@ -695,7 +695,8 @@ def test_scan_barcode_success(self): scan_info = { "sample_barcode": self.TEST_BARCODE, "sample_status": "sample-is-valid", - "technician_notes": "" + "technician_notes": "", + "observations": [] } input_json = json.dumps(scan_info) diff --git a/microsetta_private_api/admin/tests/test_admin_repo.py b/microsetta_private_api/admin/tests/test_admin_repo.py index d106e0cff..40da36a7a 100644 --- a/microsetta_private_api/admin/tests/test_admin_repo.py +++ b/microsetta_private_api/admin/tests/test_admin_repo.py @@ -5,6 +5,7 @@ import psycopg2.extras from dateutil.relativedelta import relativedelta +from microsetta_private_api.exceptions import RepoException import microsetta_private_api.model.project as p from werkzeug.exceptions import Unauthorized, NotFound @@ -323,7 +324,9 @@ def make_tz_datetime(y, m, d): "barcode": test_barcode, "scan_timestamp": make_tz_datetime(2017, 7, 16), "sample_status": 'no-registered-account', - "technician_notes": "huh?" + "technician_notes": "huh?", + "observations": [{'observation_id': None, 'observation': None, + 'category': None}] } second_scan = { @@ -331,7 +334,9 @@ def make_tz_datetime(y, m, d): "barcode": test_barcode, "scan_timestamp": make_tz_datetime(2020, 12, 4), "sample_status": 'sample-is-valid', - "technician_notes": None + "technician_notes": None, + "observations": [{'observation_id': None, 'observation': None, + 'category': None}] } try: add_dummy_scan(first_scan) @@ -347,6 +352,7 @@ def make_tz_datetime(y, m, d): self.assertGreater(len(diag['projects_info']), 0) self.assertEqual(len(diag['scans_info']), 2) # order matters in the returned vals, so test that + print(diag['scans_info'][0], first_scan) self.assertEqual(diag['scans_info'][0], first_scan) self.assertEqual(diag['scans_info'][1], second_scan) self.assertEqual(diag['latest_scan'], second_scan) @@ -776,30 +782,127 @@ def test_scan_barcode_success(self): # TODO FIXME HACK: Need to build mock barcodes rather than using # these fixed ones - TEST_BARCODE = '000000001' + TEST_BARCODE = '000010860' TEST_STATUS = "sample-has-inconsistencies" TEST_NOTES = "THIS IS A UNIT TEST" admin_repo = AdminRepo(t) + with t.dict_cursor() as cur: + cur.execute("SELECT observation_id " + "FROM " + "barcodes.sample_observation_project_associations " + "WHERE project_id = 1") + observation_id = cur.fetchone() + + # check that before doing a scan, + # no scans are recorded for this + diag = admin_repo.retrieve_diagnostics_by_barcode(TEST_BARCODE) + self.assertEqual(len(diag['scans_info']), 0) + + # do a scan + admin_repo.scan_barcode( + TEST_BARCODE, + { + "sample_status": TEST_STATUS, + "technician_notes": TEST_NOTES, + "observations": observation_id + } + ) + + # show that now a scan is recorded for this barcode + diag = admin_repo.retrieve_diagnostics_by_barcode(TEST_BARCODE) + self.assertEqual(len(diag['scans_info']), 1) + first_scan = diag['scans_info'][0] + first_observation = first_scan['observations'][0] + scan_observation_id = first_observation['observation_id'] + + self.assertEqual(first_scan['technician_notes'], TEST_NOTES) + self.assertEqual(first_scan['sample_status'], TEST_STATUS) + self.assertEqual(scan_observation_id, observation_id[0]) + + def test_scan_with_no_observations(self): + with Transaction() as t: + + TEST_BARCODE = '000010860' + TEST_NOTES = "THIS IS A UNIT TEST" + TEST_STATUS = "sample-has-inconsistencies" + admin_repo = AdminRepo(t) # check that before doing a scan, no scans are recorded for this diag = admin_repo.retrieve_diagnostics_by_barcode(TEST_BARCODE) self.assertEqual(len(diag['scans_info']), 0) - # do a scan admin_repo.scan_barcode( TEST_BARCODE, { "sample_status": TEST_STATUS, - "technician_notes": TEST_NOTES + "technician_notes": TEST_NOTES, + "observations": None } ) - - # show that now a scan is recorded for this barcode diag = admin_repo.retrieve_diagnostics_by_barcode(TEST_BARCODE) - self.assertEqual(len(diag['scans_info']), 1) first_scan = diag['scans_info'][0] - self.assertEqual(first_scan['technician_notes'], TEST_NOTES) - self.assertEqual(first_scan['sample_status'], TEST_STATUS) + first_observation = first_scan['observations'][0] + scan_observation = first_observation['observation'] + self.assertEqual(scan_observation, None) + + def test_scan_with_multiple_observations(self): + with Transaction() as t: + + TEST_BARCODE = '000010860' + TEST_NOTES = "THIS IS A UNIT TEST" + TEST_STATUS = "sample-has-inconsistencies" + admin_repo = AdminRepo(t) + + with t.dict_cursor() as cur: + cur.execute("SELECT observation_id " + "FROM " + "barcodes.sample_observation_project_associations " + "WHERE project_id = 1") + rows = cur.fetchmany(2) + observation_ids = [row['observation_id'] for row in rows] + + # check that before doing a scan, + # no scans are recorded for this + diag = admin_repo.retrieve_diagnostics_by_barcode(TEST_BARCODE) + self.assertEqual(len(diag['scans_info']), 0) + + admin_repo.scan_barcode( + TEST_BARCODE, + { + "sample_status": TEST_STATUS, + "technician_notes": TEST_NOTES, + "observations": observation_ids + } + ) + diag = admin_repo.retrieve_diagnostics_by_barcode(TEST_BARCODE) + scans = [scan['observations'] for scan in diag['scans_info']] + scans_observation_ids = [obs['observation_id'] for scan in + scans for obs in scan] + + self.assertEqual(scans_observation_ids, observation_ids) + + def test_scan_with_wrong_observation(self): + with Transaction() as t: + + TEST_BARCODE = '000000001' + TEST_NOTES = "THIS IS A UNIT TEST" + TEST_STATUS = "sample-has-inconsistencies" + TEST_OBSERVATIONS = ["ad374d60-466d-4db0-9a91-5e3e8aec7698"] + admin_repo = AdminRepo(t) + + # check that before doing a scan, no scans are recorded for this + diag = admin_repo.retrieve_diagnostics_by_barcode(TEST_BARCODE) + self.assertEqual(len(diag['scans_info']), 0) + + with self.assertRaises(RepoException): + admin_repo.scan_barcode( + TEST_BARCODE, + { + "sample_status": TEST_STATUS, + "technician_notes": TEST_NOTES, + "observations": TEST_OBSERVATIONS + } + ) def test_scan_barcode_error_nonexistent(self): with Transaction() as t: diff --git a/microsetta_private_api/api/microsetta_private_api.yaml b/microsetta_private_api/api/microsetta_private_api.yaml index 1e58da7f6..88c96e8dc 100644 --- a/microsetta_private_api/api/microsetta_private_api.yaml +++ b/microsetta_private_api/api/microsetta_private_api.yaml @@ -2546,6 +2546,25 @@ paths: '401': $ref: '#/components/responses/401Unauthorized' + '/admin/scan/observations/{sample_barcode}': + get: + operationId: microsetta_private_api.admin.admin_impl.get_observations + tags: + - Admin + parameters: + - $ref: '#/components/parameters/sample_barcode' + summary: Return a list of observations + description: Return a list of observations + responses: + '200': + description: Array of observations + content: + application/json: + schema: + type: array + '401': + $ref: '#/components/responses/401Unauthorized' + '/admin/scan/{sample_barcode}': post: # Note: We might want to be able to differentiate system administrator operations @@ -2578,6 +2597,11 @@ paths: technician_notes: type: string example: "Sample Processing Complete!" + observations: + type: array + items: + type: string + example: ["Observation 1", "Observation 2"] responses: '201': description: Successfully recorded new barcode scan diff --git a/microsetta_private_api/api/tests/test_api.py b/microsetta_private_api/api/tests/test_api.py index d65b8bb5c..8768ee163 100644 --- a/microsetta_private_api/api/tests/test_api.py +++ b/microsetta_private_api/api/tests/test_api.py @@ -2328,7 +2328,8 @@ def test_associate_sample_locked(self): any_status = 'sample-has-inconsistencies' post_resp = self.client.post('/api/admin/scan/%s' % BARCODE, json={'sample_status': any_status, - 'technician_notes': "foobar"}, + 'technician_notes': "foobar", + 'observations': []}, headers=make_headers(FAKE_TOKEN_ADMIN)) self.assertEqual(201, post_resp.status_code) @@ -2383,7 +2384,8 @@ def test_edit_sample_locked(self): bad_status = 'sample-has-inconsistencies' post_resp = self.client.post('/api/admin/scan/%s' % BARCODE, json={'sample_status': bad_status, - 'technician_notes': "foobar"}, + 'technician_notes': "foobar", + 'observations': []}, headers=make_headers(FAKE_TOKEN_ADMIN)) self.assertEqual(201, post_resp.status_code) @@ -2448,7 +2450,8 @@ def test_edit_sample_locked(self): good_status = "sample-is-valid" post_resp = self.client.post('/api/admin/scan/%s' % BARCODE, json={'sample_status': good_status, - 'technician_notes': "foobar"}, + 'technician_notes': "foobar", + 'observations': []}, headers=make_headers(FAKE_TOKEN_ADMIN)) self.assertEqual(201, post_resp.status_code) @@ -2508,7 +2511,8 @@ def test_dissociate_sample_from_source_locked(self): dummy_is_admin=True) post_resp = self.client.post('/api/admin/scan/%s' % BARCODE, json={'sample_status': 'sample-is-valid', - 'technician_notes': "foobar"}, + 'technician_notes': "foobar", + 'observations': []}, headers=make_headers(FAKE_TOKEN_ADMIN)) self.assertEqual(201, post_resp.status_code) @@ -2563,7 +2567,8 @@ def test_update_sample_association_locked(self): dummy_is_admin=True) post_resp = self.client.post('/api/admin/scan/%s' % BARCODE, json={'sample_status': 'sample-is-valid', - 'technician_notes': "foobar"}, + 'technician_notes': "foobar", + 'observations': []}, headers=make_headers(FAKE_TOKEN_ADMIN)) self.assertEqual(201, post_resp.status_code) diff --git a/microsetta_private_api/db/patches/0141.sql b/microsetta_private_api/db/patches/0141.sql new file mode 100644 index 000000000..d328dd113 --- /dev/null +++ b/microsetta_private_api/db/patches/0141.sql @@ -0,0 +1,57 @@ +-- May 13, 2024 +-- Create table to store observation categories +CREATE TABLE barcodes.sample_observation_categories ( + category VARCHAR(255) PRIMARY KEY +); + +-- Insert predefined observation categories +INSERT INTO barcodes.sample_observation_categories (category) +VALUES ('Sample'), ('Swab'), ('Tube'); + +-- Create table to store sample observations +CREATE TABLE barcodes.sample_observations ( + observation_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + category VARCHAR(255) NOT NULL, + observation VARCHAR(255) NOT NULL, + FOREIGN KEY (category) REFERENCES barcodes.sample_observation_categories(category), + UNIQUE (category, observation) +); + +-- Create table to store associations between observations and projects +CREATE TABLE barcodes.sample_observation_project_associations ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + observation_id UUID NOT NULL, + project_id INT NOT NULL, + FOREIGN KEY (observation_id) REFERENCES barcodes.sample_observations(observation_id), + FOREIGN KEY (project_id) REFERENCES barcodes.project(project_id), + UNIQUE (observation_id, project_id) +); + +-- Insert predefined observations and associate them with a project +WITH inserted_observations AS ( + INSERT INTO barcodes.sample_observations (category, observation) + VALUES + ('Tube', 'Tube is not intact'), + ('Tube', 'Screw cap is loose'), + ('Tube', 'Insufficient ethanol'), + ('Tube', 'No ethanol'), + ('Swab', 'No swab in tube'), + ('Swab', 'Multiple swabs in tube'), + ('Swab', 'Incorrect swab type'), + ('Sample', 'No visible sample'), + ('Sample', 'Excess sample on swab') + RETURNING observation_id, category, observation +) +INSERT INTO barcodes.sample_observation_project_associations (observation_id, project_id) +SELECT observation_id, 1 +FROM inserted_observations; + +-- Create table to store observation ids associated with barcode scans ids +CREATE TABLE barcodes.sample_barcode_scan_observations ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + barcode_scan_id UUID NOT NULL, + observation_id UUID NOT NULL, + FOREIGN KEY (barcode_scan_id) REFERENCES barcodes.barcode_scans(barcode_scan_id), + FOREIGN KEY (observation_id) REFERENCES barcodes.sample_observations(observation_id), + UNIQUE (barcode_scan_id, observation_id) +); diff --git a/microsetta_private_api/repo/admin_repo.py b/microsetta_private_api/repo/admin_repo.py index d3a56f0c9..5770c5433 100644 --- a/microsetta_private_api/repo/admin_repo.py +++ b/microsetta_private_api/repo/admin_repo.py @@ -376,7 +376,8 @@ def _rows_to_dicts_list(rows): # get (partial) projects_info list for this barcode query = f""" SELECT {p.DB_PROJ_NAME_KEY}, {p.IS_MICROSETTA_KEY}, - {p.BANK_SAMPLES_KEY}, {p.PLATING_START_DATE_KEY} + {p.BANK_SAMPLES_KEY}, {p.PLATING_START_DATE_KEY}, + project_id FROM barcodes.project INNER JOIN barcodes.project_barcode USING (project_id) @@ -389,13 +390,37 @@ def _rows_to_dicts_list(rows): # get scans_info list for this barcode # NB: ORDER MATTERS here. Do not change the order unless you # are positive you know what already depends on it. - cur.execute("SELECT barcode_scan_id, barcode, " - "scan_timestamp, sample_status, " - "technician_notes " - "FROM barcodes.barcode_scans " - "WHERE barcode=%s " - "ORDER BY scan_timestamp asc", - (sample_barcode,)) + cur.execute(""" + SELECT + bs.barcode_scan_id, + bs.barcode, + bs.scan_timestamp, + bs.sample_status, + bs.technician_notes, + json_agg(json_build_object('observation_id', + so.observation_id, 'observation', + so.observation, 'category', so.category)) + AS observations + FROM + barcodes.barcode_scans bs + LEFT JOIN + barcodes.sample_barcode_scan_observations bso + ON bs.barcode_scan_id = bso.barcode_scan_id + LEFT JOIN + barcodes.sample_observations so + ON bso.observation_id = so.observation_id + LEFT JOIN + barcodes.sample_observation_project_associations sopa + ON so.observation_id = sopa.observation_id + WHERE + bs.barcode = %s + GROUP BY + bs.barcode_scan_id, bs.barcode, bs.scan_timestamp, + bs.sample_status, bs.technician_notes + ORDER BY + bs.scan_timestamp ASC + """, (sample_barcode,)) + # this can't be None; worst-case is an empty list scans_info = _rows_to_dicts_list(cur.fetchall()) @@ -443,6 +468,24 @@ def _rows_to_dicts_list(rows): return diagnostic + def _rows_to_dicts_list(rows): + return [dict(x) for x in rows] + + def retrieve_observations_by_project(self, sample_barcode): + with self._transaction.dict_cursor() as cur: + cur.execute(""" + SELECT DISTINCT ON (so.observation_id) so.*, sopa.project_id + FROM barcodes.sample_observations so + JOIN barcodes.sample_observation_project_associations sopa + ON so.observation_id = sopa.observation_id + JOIN barcodes.project_barcode pb + ON sopa.project_id = pb.project_id + WHERE pb.barcode = %s + ORDER BY so.observation_id, sopa.project_id + """, (sample_barcode,)) + observations = __class__._rows_to_dicts_list(cur.fetchall()) + return observations + def get_project_name(self, project_id): """Obtain the name of a project using the project_id @@ -1069,7 +1112,6 @@ def retrieve_diagnostics_by_email(self, email): def scan_barcode(self, sample_barcode, scan_info): with self._transaction.cursor() as cur: - # not actually using the result, just checking there IS one # to ensure this is a valid barcode cur.execute( @@ -1102,6 +1144,53 @@ def scan_barcode(self, sample_barcode, scan_info): scan_args ) + if scan_info['observations']: + for observation in scan_info['observations']: + cur.execute( + "SELECT observation_id FROM sample_observations " + "WHERE observation_id = %s", + (observation,) + ) + + result = cur.fetchone() + if result is None: + raise RepoException( + f"No observation_id found for " + f"observation: {observation}" + ) + + observation_id = result[0] + + cur.execute( + """ + SELECT so.observation_id + FROM barcodes.sample_observations so + JOIN barcodes.sample_observation_project_associations + sopa ON so.observation_id = sopa.observation_id + JOIN project_barcode pb + ON sopa.project_id = pb.project_id + WHERE so.observation_id = %s AND pb.barcode = %s + """, + (observation_id, sample_barcode) + ) + + result = cur.fetchone() + if result is None: + raise RepoException( + f"Observation {observation} is not associated with" + "any project for the given barcode" + "{sample_barcode}" + ) + + cur.execute( + """ + INSERT INTO sample_barcode_scan_observations + (barcode_scan_id, observation_id) + VALUES (%s, %s) + """, + (new_uuid, observation_id) + ) + return new_uuid def search_barcode(self, sql_cond, cond_params):