From 8c716ba8582f72555e142b5cbe6fcbb2ca39b7f4 Mon Sep 17 00:00:00 2001 From: jpuerto-psc <68066250+jpuerto-psc@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:38:02 -0400 Subject: [PATCH] testing HuBMAP constraints endpoint (#1355) * testing HuBMAP constraints endpoint * updating changelog * fixing release number * General: Update CHANGELOG to "release" v0.0.24 --------- Co-authored-by: Gesina Phillips Co-authored-by: Juan Puerto <=> --- CHANGELOG.md | 3 ++- .../README.md | 2 +- .../good-cedar-multi-assay-visium/README.md | 6 +++--- .../directory-schemas/celldive-v2.0.yaml | 6 +++--- .../directory-schemas/mibi-v2.1.yaml | 6 +++--- .../directory-schemas/visium-no-probes-v2.2.yaml | 4 ++-- .../directory-schemas/visium-no-probes-v3.0.yaml | 4 ++-- .../directory-schemas/visium-no-probes-v3.1.yaml | 4 ++-- .../directory-schemas/visium-with-probes-v3.0.yaml | 4 ++-- .../directory-schemas/visium-with-probes-v3.1.yaml | 4 ++-- src/ingest_validation_tools/schema_loader.py | 2 +- 11 files changed, 23 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b4235fc2..ea7d39bc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## v0.0.24 (in progress) +## v0.0.24 - Release MERFISH - Add MERFISH directory schema - Fix documentation issue for MERFISH @@ -17,6 +17,7 @@ - Update Visium no probes directory schema - Update Visium with probes directory schema - Update Visium no probes directory schema +- Change to EntityTypeInfo constraint format to support constraints endpoint ## v0.0.23 - Add token to validation_utils.get_assaytype_data, replace URL string concatenation with urllib diff --git a/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-dir-structure/README.md b/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-dir-structure/README.md index 25e2f7577..23c54c7f9 100644 --- a/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-dir-structure/README.md +++ b/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-dir-structure/README.md @@ -1,7 +1,7 @@ ``` Directory Errors: ? examples/dataset-examples/bad-cedar-multi-assay-visium-bad-dir-structure/upload/Visium_9OLC_A4_S1 - (as visium-no-probes-v2.1) + (as visium-no-probes-v2.2) : - Required but missing: - lab_processed\/.*. - lab_processed\/images\/.*. diff --git a/examples/dataset-examples/good-cedar-multi-assay-visium/README.md b/examples/dataset-examples/good-cedar-multi-assay-visium/README.md index f9d0d9aeb..1ccf5c9a7 100644 --- a/examples/dataset-examples/good-cedar-multi-assay-visium/README.md +++ b/examples/dataset-examples/good-cedar-multi-assay-visium/README.md @@ -7,14 +7,14 @@ TSVs: good-visium-assay-metadata.tsv: Schema: visium-no-probes-v2 Metadata schema version: '2' - Directory schema version: visium-no-probes-v2.1 + Directory schema version: visium-no-probes-v2.2 good-visium-histology-metadata.tsv: Schema: h-and-e-v2 Metadata schema version: '2' - Directory schema version: visium-no-probes-v2.1 + Directory schema version: visium-no-probes-v2.2 good-visium-rnaseq-metadata.tsv: Schema: rnaseq-visium-no-probes-v2 Metadata schema version: '2' - Directory schema version: visium-no-probes-v2.1 + Directory schema version: visium-no-probes-v2.2 ``` \ No newline at end of file diff --git a/src/ingest_validation_tools/directory-schemas/celldive-v2.0.yaml b/src/ingest_validation_tools/directory-schemas/celldive-v2.0.yaml index 0e4447131..2ff10e7c9 100644 --- a/src/ingest_validation_tools/directory-schemas/celldive-v2.0.yaml +++ b/src/ingest_validation_tools/directory-schemas/celldive-v2.0.yaml @@ -13,15 +13,15 @@ files: required: False description: A file generated by the micro-meta app that contains a description of the settings that were used to acquire the image data. Email HuBMAP Consortium Help Desk if help is required in generating this document. is_qa_qc: True - - + - pattern: raw\/.* required: True description: This is a directory containing raw data. - - + - pattern: raw\/images\/.* required: True description: Raw image files. Using this subdirectory allows for harmonization with other more complex assays, like Visium that includes both raw imaging and sequencing data. - - + - pattern: raw\/images\/round_info_[^\/]+\.dat required: True description: Metadata file for the capture item-value tab separated format. This contains various instrument and acquisition details for each acquisition cycle. diff --git a/src/ingest_validation_tools/directory-schemas/mibi-v2.1.yaml b/src/ingest_validation_tools/directory-schemas/mibi-v2.1.yaml index f3d682cbf..f71ccc6cc 100644 --- a/src/ingest_validation_tools/directory-schemas/mibi-v2.1.yaml +++ b/src/ingest_validation_tools/directory-schemas/mibi-v2.1.yaml @@ -13,11 +13,11 @@ files: required: False description: A file generated by the micro-meta app that contains a description of the settings that were used to acquire the image data. Email HuBMAP Consortium Help Desk if help is required in generating this document. is_qa_qc: True - - + - pattern: raw\/.* required: True - description: This is a directory containing raw data. - - + description: This is a directory containing raw data. + - pattern: raw\/images\/.* required: True description: Raw image files. Using this subdirectory allows for harmonization with other more complex assays, like Visium that includes both raw imaging and sequencing data. diff --git a/src/ingest_validation_tools/directory-schemas/visium-no-probes-v2.2.yaml b/src/ingest_validation_tools/directory-schemas/visium-no-probes-v2.2.yaml index 40fd4021c..67dbfee11 100644 --- a/src/ingest_validation_tools/directory-schemas/visium-no-probes-v2.2.yaml +++ b/src/ingest_validation_tools/directory-schemas/visium-no-probes-v2.2.yaml @@ -26,7 +26,7 @@ files: pattern: raw\/fastq\/.* required: True description: Raw sequencing files for the experiment - - + - pattern: raw\/fastq\/RNA\/.* required: True description: Directory containing fastq files pertaining to RNAseq sequencing. @@ -74,5 +74,5 @@ files: - pattern: lab_processed\/transformations\/[^\/]+\.txt required: False - description: Transformation matrices used to overlay images from the exact same tissue section (e.g., MALDI imaging mass spec, autofluorescence microscopy, MxIF, histological stains). + description: Transformation matrices used to overlay images from the exact same tissue section (e.g., MALDI imaging mass spec, autofluorescence microscopy, MxIF, histological stains). is_qa_qc: False diff --git a/src/ingest_validation_tools/directory-schemas/visium-no-probes-v3.0.yaml b/src/ingest_validation_tools/directory-schemas/visium-no-probes-v3.0.yaml index e7e7ce873..ac3430bdf 100644 --- a/src/ingest_validation_tools/directory-schemas/visium-no-probes-v3.0.yaml +++ b/src/ingest_validation_tools/directory-schemas/visium-no-probes-v3.0.yaml @@ -26,7 +26,7 @@ files: pattern: raw\/fastq\/.* required: True description: Raw sequencing files for the experiment - - + - pattern: raw\/fastq\/RNA\/.* required: True description: Directory containing fastq files pertaining to RNAseq sequencing. @@ -78,5 +78,5 @@ files: - pattern: lab_processed\/transformations\/[^\/]+\.txt required: False - description: Transformation matrices used to overlay images from the exact same tissue section (e.g., MALDI imaging mass spec, autofluorescence microscopy, MxIF, histological stains). + description: Transformation matrices used to overlay images from the exact same tissue section (e.g., MALDI imaging mass spec, autofluorescence microscopy, MxIF, histological stains). is_qa_qc: False diff --git a/src/ingest_validation_tools/directory-schemas/visium-no-probes-v3.1.yaml b/src/ingest_validation_tools/directory-schemas/visium-no-probes-v3.1.yaml index e5c7ba9f6..a50049b90 100644 --- a/src/ingest_validation_tools/directory-schemas/visium-no-probes-v3.1.yaml +++ b/src/ingest_validation_tools/directory-schemas/visium-no-probes-v3.1.yaml @@ -26,7 +26,7 @@ files: pattern: raw\/fastq\/.* required: True description: Raw sequencing files for the experiment - - + - pattern: raw\/fastq\/RNA\/.* required: True description: Directory containing fastq files pertaining to RNAseq sequencing. @@ -74,5 +74,5 @@ files: - pattern: lab_processed\/transformations\/[^\/]+\.txt required: False - description: Transformation matrices used to overlay images from the exact same tissue section (e.g., MALDI imaging mass spec, autofluorescence microscopy, MxIF, histological stains). + description: Transformation matrices used to overlay images from the exact same tissue section (e.g., MALDI imaging mass spec, autofluorescence microscopy, MxIF, histological stains). is_qa_qc: False diff --git a/src/ingest_validation_tools/directory-schemas/visium-with-probes-v3.0.yaml b/src/ingest_validation_tools/directory-schemas/visium-with-probes-v3.0.yaml index f973a3d1f..69ccadf6d 100644 --- a/src/ingest_validation_tools/directory-schemas/visium-with-probes-v3.0.yaml +++ b/src/ingest_validation_tools/directory-schemas/visium-with-probes-v3.0.yaml @@ -22,11 +22,11 @@ files: required: True description: This is a 10X Genomics layout file that's generated by 10X and individualized for each Visium slide. This is a text file and can be generated using this 10X web form along with the unique 10X Visium slide ID. is_qa_qc: False - - + - pattern: raw\/additional_panels_used\.csv required: False description: If multiple commercial probe panels were used, then the primary probe panel should be selected in the "oligo_probe_panel" metadata field. The additional panels must be included in this file. Each panel record should include:manufacturer, model/name, product code. - - + - pattern: raw\/custom_probe_set\.csv required: False description: This file should contain any custom probes used and must be included if the metadata field "is_custom_probes_used" is "Yes". The file should minimally include:target gene id, probe seq, probe id. The contents of this file are modeled after the 10x Genomics probe set file (see ). diff --git a/src/ingest_validation_tools/directory-schemas/visium-with-probes-v3.1.yaml b/src/ingest_validation_tools/directory-schemas/visium-with-probes-v3.1.yaml index b761dd4e4..be1cbdfe5 100644 --- a/src/ingest_validation_tools/directory-schemas/visium-with-probes-v3.1.yaml +++ b/src/ingest_validation_tools/directory-schemas/visium-with-probes-v3.1.yaml @@ -22,11 +22,11 @@ files: required: True description: This is a 10X Genomics layout file that's generated by 10X and individualized for each Visium slide. This is a text file and can be generated using this 10X web form along with the unique 10X Visium slide ID. is_qa_qc: False - - + - pattern: raw\/additional_panels_used\.csv required: False description: If multiple commercial probe panels were used, then the primary probe panel should be selected in the "oligo_probe_panel" metadata field. The additional panels must be included in this file. Each panel record should include:manufacturer, model/name, product code. - - + - pattern: raw\/custom_probe_set\.csv required: False description: This file should contain any custom probes used and must be included if the metadata field "is_custom_probes_used" is "Yes". The file should minimally include:target gene id, probe seq, probe id. The contents of this file are modeled after the 10x Genomics probe set file (see ). diff --git a/src/ingest_validation_tools/schema_loader.py b/src/ingest_validation_tools/schema_loader.py index 5d602d8cb..8018826d6 100644 --- a/src/ingest_validation_tools/schema_loader.py +++ b/src/ingest_validation_tools/schema_loader.py @@ -141,7 +141,7 @@ def format_constraint_check_data(self) -> Dict: """ return { "entity_type": self.entity_type.value, - "sub_type": [self.entity_sub_type if self.entity_sub_type else ""], + "sub_type": [self.entity_sub_type] if self.entity_sub_type else None, "sub_type_val": [self.entity_sub_type_val] if self.entity_sub_type_val else None, }