diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c159910d..f9100e19c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Ported murine from SenNet - Update Histology directory schema - Bugfix stripping trailing slash in ingest api url +- Converted upload `_url_checks` to use `_get_method` for SenNet compatibility ## v0.0.17 diff --git a/examples/dataset-examples/bad-cedar-assay-histology/README.md b/examples/dataset-examples/bad-cedar-assay-histology/README.md index 68ec147aa..adce717b0 100644 --- a/examples/dataset-examples/bad-cedar-assay-histology/README.md +++ b/examples/dataset-examples/bad-cedar-assay-histology/README.md @@ -5,22 +5,21 @@ Upload Errors: column 'contributors_path', value './contributors.tsv' : CEDAR Validation Errors: examples/dataset-examples/bad-cedar-assay-histology/upload/contributors.tsv: - examples/dataset-examples/bad-cedar-assay-histology/upload/contributors.tsv: - - Validation Errors: - - On row 0, column "orcid", value "0000-0002-8928-abcd" fails because - of error "invalidValueFormat". + Validation Errors: + - On row 0, column "orcid", value "0000-0002-8928-abcd" fails because of + error "invalidValueFormat". Metadata TSV Validation Errors: CEDAR Validation Errors: examples/dataset-examples/bad-cedar-assay-histology/upload/bad-histology-metadata.tsv: - examples/dataset-examples/bad-cedar-assay-histology/upload/bad-histology-metadata.tsv: - - URL Errors: - - 'Row 2, field "parent_sample_id" with value "wrong": Field value is not - valid; URL https://entity.api.hubmapconsortium.org/entities/wrong returned - a 400 Error.' - Validation Errors: - - On row 0, column "parent_sample_id", value "wrong" fails because of error - "invalidValueFormat". - - On row 1, column "contributors_path", value "" fails because of error "missingRequired". + URL Errors: + - 'On row 2, column "parent_sample_id", value "wrong" fails because of error + "HTTPError": 401 Client Error: Unauthorized for url: https://entity.api.hubmapconsortium.org/entities/wrong.' + - 'On row 3, column "parent_sample_id", value "HBM854.FXDQ.783" fails because + of error "HTTPError": 401 Client Error: Unauthorized for url: https://entity.api.hubmapconsortium.org/entities/HBM854.FXDQ.783' + Validation Errors: + - On row 0, column "parent_sample_id", value "wrong" fails because of error + "invalidValueFormat". + - On row 1, column "contributors_path", value "" fails because of error "missingRequired". Reference Errors: No References: Files: @@ -28,4 +27,4 @@ Reference Errors: Hint: 'If validation fails because of extra whitespace in the TSV, try: src/cleanup_whitespace.py --tsv_in original.tsv --tsv_out clean.tsv.' -``` +``` \ No newline at end of file diff --git a/examples/dataset-examples/bad-cedar-assay-histology/fixtures.json b/examples/dataset-examples/bad-cedar-assay-histology/fixtures.json index d36b800d7..81e9098a0 100644 --- a/examples/dataset-examples/bad-cedar-assay-histology/fixtures.json +++ b/examples/dataset-examples/bad-cedar-assay-histology/fixtures.json @@ -1 +1 @@ -{"assaytype": {"Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}}, "validation": {"h-and-e": {"examples/dataset-examples/bad-cedar-assay-histology/upload/bad-histology-metadata.tsv": [{"URL Errors": ["Row 2, field 'parent_sample_id' with value 'wrong': 400 Client Error: Bad Request for url: https://entity.api.hubmapconsortium.org/entities/wrong"], "Validation Errors": ["On row 0, column \"parent_sample_id\", value \"wrong\" fails because of error \"invalidValueFormat\"", "On row 1, column \"contributors_path\", value \"\" fails because of error \"missingRequired\""]}]}, "contributors": {"examples/dataset-examples/bad-cedar-assay-histology/upload/contributors.tsv": [{"Validation Errors": ["On row 0, column \"orcid\", value \"0000-0002-8928-abcd\" fails because of error \"invalidValueFormat\""]}]}}} \ No newline at end of file +{"assaytype": {"Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}}, "validation": {"h-and-e": {"URL Errors": ["On row 2, column \"parent_sample_id\", value \"wrong\" fails because of error \"HTTPError\": 400 Client Error: Bad Request for url: https://entity.api.hubmapconsortium.org/entities/wrong"], "Validation Errors": ["On row 0, column \"parent_sample_id\", value \"wrong\" fails because of error \"invalidValueFormat\"", "On row 1, column \"contributors_path\", value \"\" fails because of error \"missingRequired\""]}, "contributors": {"Validation Errors": ["On row 0, column \"orcid\", value \"0000-0002-8928-abcd\" fails because of error \"invalidValueFormat\""]}}} \ No newline at end of file diff --git a/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/README.md b/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/README.md index 87242b9d6..28bf400b4 100644 --- a/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/README.md +++ b/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/README.md @@ -2,14 +2,13 @@ Metadata TSV Validation Errors: CEDAR Validation Errors: examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/upload/bad-visium-rnaseq-metadata.tsv: - examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/upload/bad-visium-rnaseq-metadata.tsv: - - URL Errors: - - 'Row 3, field "parent_sample_id" with value "": 404 Client Error: Not Found - for url: https://entity.api.hubmapconsortium.org/entities/.' - Validation Errors: - - On row 1, column "parent_sample_id", value "" fails because of error "missingRequired". - - On row 2, column "preparation_protocol_doi", value "wrong" fails because - of error "invalidUrl". + URL Errors: + - 'On row 3, column "parent_sample_id", value "" fails because of error "HTTPError": + 404 Client Error: Not Found for url: https://entity.api.hubmapconsortium.org/entities/.' + Validation Errors: + - On row 1, column "parent_sample_id", value "" fails because of error "missingRequired". + - On row 2, column "preparation_protocol_doi", value "wrong" fails because of + error "invalidUrl". Reference Errors: No References: Files: @@ -17,4 +16,4 @@ Reference Errors: Hint: 'If validation fails because of extra whitespace in the TSV, try: src/cleanup_whitespace.py --tsv_in original.tsv --tsv_out clean.tsv.' -``` +``` \ No newline at end of file diff --git a/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/fixtures.json b/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/fixtures.json index f7e3a7e21..bb78feb74 100644 --- a/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/fixtures.json +++ b/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/fixtures.json @@ -1 +1 @@ -{"assaytype": {"RNAseq": {"assaytype": "scRNAseq-10Genomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}, "Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (No probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}}, "validation": {"scRNAseq-10Genomics-v3": {"examples/dataset-examples/bad-cedar-multi-assay-visium-bad-child-metadata/upload/bad-visium-rnaseq-metadata.tsv": [{"URL Errors": ["Row 3, field 'parent_sample_id' with value '': 404 Client Error: Not Found for url: https://entity.api.hubmapconsortium.org/entities/"], "Validation Errors": ["On row 1, column \"parent_sample_id\", value \"\" fails because of error \"missingRequired\"", "On row 2, column \"preparation_protocol_doi\", value \"wrong\" fails because of error \"invalidUrl\""]}]}, "contributors": null, "visium-no-probes": null, "h-and-e": null}} \ No newline at end of file +{"assaytype": {"RNAseq": {"assaytype": "scRNAseq-10xGenomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}, "Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (no probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}}, "validation": {"scRNAseq-10xGenomics-v3": {"URL Errors": ["On row 3, column \"parent_sample_id\", value \"\" fails because of error \"HTTPError\": 404 Client Error: Not Found for url: https://entity.api.hubmapconsortium.org/entities/"], "Validation Errors": ["On row 1, column \"parent_sample_id\", value \"\" fails because of error \"missingRequired\"", "On row 2, column \"preparation_protocol_doi\", value \"wrong\" fails because of error \"invalidUrl\""]}, "contributors": null, "visium-no-probes": null, "h-and-e": null}} \ No newline at end of file diff --git a/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-dir-structure/fixtures.json b/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-dir-structure/fixtures.json index c56029912..4c99b98f8 100644 --- a/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-dir-structure/fixtures.json +++ b/examples/dataset-examples/bad-cedar-multi-assay-visium-bad-dir-structure/fixtures.json @@ -1 +1 @@ -{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (No probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10Genomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {"visium-no-probes": null, "contributors": null, "h-and-e": null, "scRNAseq-10Genomics-v3": null}} \ No newline at end of file +{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (no probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10xGenomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {"visium-no-probes": null, "contributors": null, "h-and-e": null, "scRNAseq-10xGenomics-v3": null}} \ No newline at end of file diff --git a/examples/dataset-examples/bad-cedar-multi-assay-visium-missing-child/fixtures.json b/examples/dataset-examples/bad-cedar-multi-assay-visium-missing-child/fixtures.json index 78f6d8876..be91292c0 100644 --- a/examples/dataset-examples/bad-cedar-multi-assay-visium-missing-child/fixtures.json +++ b/examples/dataset-examples/bad-cedar-multi-assay-visium-missing-child/fixtures.json @@ -1 +1 @@ -{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (No probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}}, "validation": {}} \ No newline at end of file +{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (no probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}}, "validation": {}} \ No newline at end of file diff --git a/examples/dataset-examples/bad-cedar-multi-assay-visium-unreferenced-parent-path/fixtures.json b/examples/dataset-examples/bad-cedar-multi-assay-visium-unreferenced-parent-path/fixtures.json index 788b23491..b22ffa40f 100644 --- a/examples/dataset-examples/bad-cedar-multi-assay-visium-unreferenced-parent-path/fixtures.json +++ b/examples/dataset-examples/bad-cedar-multi-assay-visium-unreferenced-parent-path/fixtures.json @@ -1 +1 @@ -{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (No probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10Genomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {}} \ No newline at end of file +{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (no probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10xGenomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {}} \ No newline at end of file diff --git a/examples/dataset-examples/bad-cedar-multi-assay-visium-with-standalone-histology/fixtures.json b/examples/dataset-examples/bad-cedar-multi-assay-visium-with-standalone-histology/fixtures.json index 788b23491..b22ffa40f 100644 --- a/examples/dataset-examples/bad-cedar-multi-assay-visium-with-standalone-histology/fixtures.json +++ b/examples/dataset-examples/bad-cedar-multi-assay-visium-with-standalone-histology/fixtures.json @@ -1 +1 @@ -{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (No probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10Genomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {}} \ No newline at end of file +{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (no probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10xGenomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {}} \ No newline at end of file diff --git a/examples/dataset-examples/bad-cedar-multi-assay-visium-wrong-child/fixtures.json b/examples/dataset-examples/bad-cedar-multi-assay-visium-wrong-child/fixtures.json index d3b154fe8..47773ae03 100644 --- a/examples/dataset-examples/bad-cedar-multi-assay-visium-wrong-child/fixtures.json +++ b/examples/dataset-examples/bad-cedar-multi-assay-visium-wrong-child/fixtures.json @@ -1 +1 @@ -{"assaytype": {"RNAseq (with probes)": {"assaytype": "scRNAseq-visium-with-probes", "contains-pii": true, "dataset-type": "RNAseq (with probes)", "description": "Visium RNAseq with probes", "dir-schema": "rnaseq-with-probes-v2", "primary": true, "vitessce-hints": []}, "Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (No probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}}, "validation": {}} \ No newline at end of file +{"assaytype": {"RNAseq (with probes)": {"assaytype": "scRNAseq-visium-with-probes", "contains-pii": true, "dataset-type": "RNAseq (with probes)", "description": "Visium RNAseq with probes", "dir-schema": "rnaseq-with-probes-v2", "primary": true, "vitessce-hints": []}, "Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (no probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}}, "validation": {}} \ No newline at end of file diff --git a/examples/dataset-examples/good-cedar-multi-assay-visium/README.md b/examples/dataset-examples/good-cedar-multi-assay-visium/README.md index 99e56e681..e7cf8a2ba 100644 --- a/examples/dataset-examples/good-cedar-multi-assay-visium/README.md +++ b/examples/dataset-examples/good-cedar-multi-assay-visium/README.md @@ -13,7 +13,7 @@ TSVs: Metadata schema version: '2' Directory schema versions: histology-v2 good-visium-rnaseq-metadata.tsv: - Schema: scRNAseq-10Genomics-v3-v2 + Schema: scRNAseq-10xGenomics-v3-v2 Metadata schema version: '2' Directory schema versions: rnaseq-v2 diff --git a/examples/dataset-examples/good-cedar-multi-assay-visium/fixtures.json b/examples/dataset-examples/good-cedar-multi-assay-visium/fixtures.json index c56029912..4c99b98f8 100644 --- a/examples/dataset-examples/good-cedar-multi-assay-visium/fixtures.json +++ b/examples/dataset-examples/good-cedar-multi-assay-visium/fixtures.json @@ -1 +1 @@ -{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (No probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10Genomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {"visium-no-probes": null, "contributors": null, "h-and-e": null, "scRNAseq-10Genomics-v3": null}} \ No newline at end of file +{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (no probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10xGenomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {"visium-no-probes": null, "contributors": null, "h-and-e": null, "scRNAseq-10xGenomics-v3": null}} \ No newline at end of file diff --git a/examples/plugin-tests/expected-failure/fixtures.json b/examples/plugin-tests/expected-failure/fixtures.json index c56029912..4c99b98f8 100644 --- a/examples/plugin-tests/expected-failure/fixtures.json +++ b/examples/plugin-tests/expected-failure/fixtures.json @@ -1 +1 @@ -{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (No probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10Genomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {"visium-no-probes": null, "contributors": null, "h-and-e": null, "scRNAseq-10Genomics-v3": null}} \ No newline at end of file +{"assaytype": {"Visium (no probes)": {"assaytype": "visium-no-probes", "contains-pii": true, "dataset-type": "Visium (no probes)", "description": "Visium (no probes)", "dir-schema": "visium-no-probes-v2", "is-multi-assay": true, "must-contain": ["Histology", "RNAseq"], "primary": true, "vitessce-hints": []}, "Histology": {"assaytype": "h-and-e", "contains-pii": false, "dataset-type": "Histology", "description": "H&E Stained Microscopy", "dir-schema": "histology-v2", "primary": true, "vitessce-hints": []}, "RNAseq": {"assaytype": "scRNAseq-10xGenomics-v3", "contains-pii": true, "dataset-type": "RNAseq", "description": "scRNA-seq (10x Genomics v3)", "dir-schema": "rnaseq-v2", "primary": true, "vitessce-hints": []}}, "validation": {"visium-no-probes": null, "contributors": null, "h-and-e": null, "scRNAseq-10xGenomics-v3": null}} \ No newline at end of file diff --git a/src/ingest_validation_tools/upload.py b/src/ingest_validation_tools/upload.py index d3ed5e236..3602b57de 100644 --- a/src/ingest_validation_tools/upload.py +++ b/src/ingest_validation_tools/upload.py @@ -334,10 +334,10 @@ def online_checks( schema_name: str, report_type: ReportType = ReportType.STR, ) -> Optional[Dict]: - url_errors = self._url_checks(tsv_path, schema_name) + url_errors = self._url_checks(tsv_path, schema_name, report_type) api_errors = self._api_validation(Path(tsv_path), report_type) if url_errors or api_errors: - return {tsv_path: [url_errors | api_errors]} + return url_errors | api_errors def _get_reference_errors(self) -> dict: errors: Dict[str, Any] = {} @@ -506,7 +506,9 @@ def _cedar_api_call(self, tsv_path: Union[str, Path]) -> requests.models.Respons raise Exception(f"CEDAR API request for {tsv_path} failed! Exception: {e}") return response - def _url_checks(self, tsv_path: str, schema_name: str): + def _url_checks( + self, tsv_path: str, schema_name: str, report_type: ReportType = ReportType.STR + ): """ Check provided UUIDs/HuBMAP IDs for parent_id, sample_id, organ_id. Not using get_table_errors because CEDAR schema fields do not match @@ -532,12 +534,14 @@ def _url_checks(self, tsv_path: str, schema_name: str): else: constrained_fields["parent_sample_id"] = self.app_context.get("entities_url") - url_errors = self._check_matching_urls(tsv_path, constrained_fields) + url_errors = self._check_matching_urls(tsv_path, constrained_fields, report_type) if url_errors: errors["URL Errors"] = url_errors return errors - def _check_matching_urls(self, tsv_path: str, constrained_fields: dict): + def _check_matching_urls( + self, tsv_path: str, constrained_fields: dict, report_type: ReportType = ReportType.STR + ): rows = read_rows(Path(tsv_path), self.encoding) fields = rows[0].keys() missing_fields = [k for k in constrained_fields.keys() if k not in fields].sort() @@ -559,7 +563,14 @@ def _check_matching_urls(self, tsv_path: str, constrained_fields: dict): response = requests.get(url, headers=headers) response.raise_for_status() except Exception as e: - url_errors.append(f"Row {i+2}, field '{field}' with value '{value}': {e}") + error = { + "errorType": type(e).__name__, + "column": field, + "row": i + 2, + "value": value, + "error_text": e.__str__(), + } + url_errors.append(self._get_message(error, report_type)) return url_errors def _get_message( @@ -584,6 +595,7 @@ def _get_message( """ # noqa: E501 example = error.get("repairSuggestion", "") + error_text = error.get("error_text", "") return_str = report_type is ReportType.STR if "errorType" in error and "column" in error and "row" in error and "value" in error: @@ -591,6 +603,7 @@ def _get_message( msg = ( f'On row {error["row"]}, column "{error["column"]}", ' f'value "{error["value"]}" fails because of error "{error["errorType"]}"' + f'{f": {error_text}" if error_text else error_text}' f'{f". Example: {example}" if example else example}' ) return msg if return_str else get_json(msg, error["row"], error["column"])