Merge branch 'main' of https://github.com/hubmapconsortium/hra-do-pro…

…cessor

er-diagrams.md

@@ -85,6 +85,7 @@ BiomarkerInstance {
 }
 AsctbConcept {
     uriorcurie id  
+    string label  
     string conforms_to  
     uriorcurie parent_class  
     string ccf_pref_label  
@@ -108,6 +109,7 @@ AnatomicalStructureInstance {
 Biomarker {
     string ccf_biomarker_type  
     uriorcurie id  
+    string label  
     string conforms_to  
     uriorcurie parent_class  
     string ccf_pref_label  
@@ -116,6 +118,7 @@ Biomarker {
 }
 AnatomicalStructure {
     uriorcurie id  
+    string label  
     string conforms_to  
     uriorcurie parent_class  
     string ccf_pref_label  
@@ -124,6 +127,7 @@ AnatomicalStructure {
 }
 CellType {
     uriorcurie id  
+    string label  
     string conforms_to  
     uriorcurie parent_class  
     string ccf_pref_label  

schemas/src/digital-objects/2d-ftu.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/2d-ftu
+id: https://purl.humanatlas.io/schema/2d-ftu
 name: ftu-2d
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/digital-objects/asct-b.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/asct-b
+id: https://purl.humanatlas.io/schema/asct-b
 name: asct-b
 prefixes:
   ccf: http://purl.org/ccf/
@@ -265,7 +265,7 @@ slots:
     required: false
     slot_uri: rdfs:label
     annotations:
-      owl: AnnotationProperty, AnnotationAssertion   
+      owl: AnnotationProperty, AnnotationAssertion
   ccf_pref_label:
     required: true
     slot_uri: ccf:ccf_pref_label

schemas/src/digital-objects/basic.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/basic
+id: https://purl.humanatlas.io/schema/basic
 name: basic
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/digital-objects/collection.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/collection
+id: https://purl.humanatlas.io/schema/collection
 name: collection
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/digital-objects/ds-graph.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/ds-graph
+id: https://purl.humanatlas.io/schema/ds-graph
 name: ds-graph
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/digital-objects/graph.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/graph
+id: https://purl.humanatlas.io/schema/graph
 name: graph
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/digital-objects/landmark.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/landmark
+id: https://purl.humanatlas.io/schema/landmark
 name: landmark
 prefixes:
   ccf: http://purl.org/ccf/
@@ -136,7 +136,7 @@ classes:
         AnnotationAssertion( prov:wasDerivedFrom {{iri}} {{metadata.derived_from|e}} )
         AnnotationAssertion( IAO:0000700 {{iri}} ccf:ExtractionSet )
         AnnotationAssertion( IAO:0000700 {{iri}} ccf:SpatialEntity )
-        
+
 
 slots:
   landmarks:

schemas/src/digital-objects/omap.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/omap
+id: https://purl.humanatlas.io/schema/omap
 name: omap
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/digital-objects/ref-organ.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/ref-organ
+id: https://purl.humanatlas.io/schema/ref-organ
 name: ref-organ
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/digital-objects/vocab.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/vocab
+id: https://purl.humanatlas.io/schema/vocab
 name: vocab
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/metadata/2d-ftu-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/2d-ftu-metadata
+id: https://purl.humanatlas.io/schema/2d-ftu-metadata
 name: ftu-2d-metadata
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/metadata/asct-b-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/asct-b-metadata
+id: https://purl.humanatlas.io/schema/asct-b-metadata
 name: asct-b-metadata
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/metadata/basic-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/basic-metadata
+id: https://purl.humanatlas.io/schema/basic-metadata
 name: basic-metadata
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/metadata/collection-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/collection-metadata
+id: https://purl.humanatlas.io/schema/collection-metadata
 name: collection-metadata
 prefixes:
   ccf: http://purl.org/ccf/
@@ -60,4 +60,4 @@ classes:
       ontology_root:
         required: false
         annotations:
-          owl: AnnotationProperty, AnnotationAssertion
+          owl: AnnotationProperty, AnnotationAssertion

schemas/src/metadata/ds-graph-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/ds-graph-metadata
+id: https://purl.humanatlas.io/schema/ds-graph-metadata
 name: ds-graph-metadata
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/metadata/graph-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/graph-metadata
+id: https://purl.humanatlas.io/schema/graph-metadata
 name: graph-metadata
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/metadata/landmark-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/landmark-metadata
+id: https://purl.humanatlas.io/schema/landmark-metadata
 name: landmark-metadata
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/metadata/omap-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/omap-metadata
+id: https://purl.humanatlas.io/schema/omap-metadata
 name: omap-metadata
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/metadata/ref-organ-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/ref-organ-metadata
+id: https://purl.humanatlas.io/schema/ref-organ-metadata
 name: ref-organ-metadata
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/metadata/vocab-metadata.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/vocab-metadata
+id: https://purl.humanatlas.io/schema/vocab-metadata
 name: vocab-metadata
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/shared/entity-base.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/entity-base
+id: https://purl.humanatlas.io/schema/entity-base
 name: entity-base
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/shared/instance-base.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/instance-base
+id: https://purl.humanatlas.io/schema/instance-base
 name: instance-base
 prefixes:
   ccf: http://purl.org/ccf/

schemas/src/shared/metadata-base.yaml

@@ -1,4 +1,4 @@
-id: https://purl.humanatlas.io/specs/metadata-base
+id: https://purl.humanatlas.io/schema/metadata-base
 name: metadata-base
 prefixes:
   ccf: http://purl.org/ccf/

src/cli.js

@@ -13,6 +13,7 @@ import { list } from './list.js';
 import { migrateCcfLandmarks } from './migration/ccf-landmarks/migrate.js';
 import { migrateCcfOwl } from './migration/ccf-owl/migrate.js';
 import { migrateCcfReleases } from './migration/ccf-releases/migrate.js';
+import { migrateSchemas } from './migration/schemas/migrate.js';
 import { normalize } from './normalization/normalize.js';
 import { updateCollection } from './update-collection.js';
 import { updateRefOrganCrosswalk } from './update-ref-organ-crosswalk.js';
@@ -138,6 +139,13 @@ program
     createDb(getContext(program, command));
   });
 
+program
+  .command('migrate-schemas')
+  .description('Migrate linkml schemas to HRA Digital Object format for publishing')
+  .action((_options, command) => {
+    migrateSchemas(getContext(program, command));
+  });
+
 program
   .command('migrate-ccf-landmarks')
   .description('Migrate ccf landmarks to HRA Digital Object format')

src/deployment/deploy.js

@@ -11,11 +11,15 @@ export function deploy(context) {
   const deployPath = resolve(context.deploymentHome, obj.doString);
   const metadata = readMetadata(context);
   const graph = resolve(deployPath, 'graph.ttl');
+  const json = resolve(deployPath, 'graph.json');
   const metadataTtl = resolve(deployPath, 'metadata.ttl');
+  const metadataJson = resolve(deployPath, 'metadata.json');
 
   sh.mkdir('-p', resolve(deployPath, 'assets'));
   sh.cp(resolve(obj.path, 'enriched/enriched.ttl'), graph);
+  sh.cp(resolve(obj.path, 'enriched/enriched.json'), json);
   sh.cp(resolve(obj.path, 'enriched/enriched-metadata.ttl'), metadataTtl);
+  sh.cp(resolve(obj.path, 'enriched/enriched-metadata.json'), metadataJson);
 
   for (const file of metadata.datatable) {
     if (file.includes('/')) {
@@ -28,7 +32,7 @@ export function deploy(context) {
     info("Removing OWL individuals from the graph output.");
     removeIndividuals(graph, graph);
   }
-  
+
   info(`Reifying "${obj.doString}"`);
   reifyDoTurtle(context, graph);
   if (context.updateDb) {

src/enrichment/enrich-2d-ftu.js

@@ -13,14 +13,18 @@ import {
   extractOntologySubset,
   excludeTerms,
   logOutput,
-  push
+  push,
+  convertNormalizedDataToJson,
+  convertNormalizedMetadataToJson
 } from './utils.js';
 
 export function enrich2dFtuMetadata(context) {
   const { selectedDigitalObject: obj } = context;
   const normalizedPath = resolve(obj.path, 'normalized/normalized-metadata.yaml');
   const enrichedPath = resolve(obj.path, 'enriched/enriched-metadata.ttl');
   convertNormalizedMetadataToRdf(context, normalizedPath, enrichedPath);
+  const enrichedJson = resolve(obj.path, 'enriched/enriched-metadata.json');
+  convertNormalizedMetadataToJson(context, normalizedPath, enrichedJson);
 }
 
 export function enrich2dFtuData(context) {
@@ -80,6 +84,9 @@ export function enrich2dFtuData(context) {
     const enrichedPath = resolve(obj.path, 'enriched/enriched.ttl');
     info(`Creating 2d-ftu: ${enrichedPath}`);
     convert(trimmedOutputPath, enrichedPath, 'ttl');
+
+    const enrichedJsonPath = resolve(obj.path, 'enriched/enriched.json');
+    convertNormalizedDataToJson(context, normalizedPath, enrichedJsonPath);
   } catch (e) {
     error(e);
   } finally {

src/enrichment/enrich-asct-b.js

@@ -5,7 +5,9 @@ import { throwOnError } from '../utils/sh-exec.js';
 import {
   cleanTemporaryFiles,
   collectEntities,
+  convertNormalizedDataToJson,
   convertNormalizedDataToOwl,
+  convertNormalizedMetadataToJson,
   convertNormalizedMetadataToRdf,
   excludeTerms,
   extractClassHierarchy,
@@ -18,6 +20,8 @@ export function enrichAsctbMetadata(context) {
   const normalizedPath = resolve(obj.path, 'normalized/normalized-metadata.yaml');
   const enrichedPath = resolve(obj.path, 'enriched/enriched-metadata.ttl');
   convertNormalizedMetadataToRdf(context, normalizedPath, enrichedPath);
+  const enrichedJson = resolve(obj.path, 'enriched/enriched-metadata.json');
+  convertNormalizedMetadataToJson(context, normalizedPath, enrichedJson);
 }
 
 export function enrichAsctbData(context) {
@@ -133,6 +137,9 @@ export function enrichAsctbData(context) {
     const enrichedPath = resolve(obj.path, 'enriched/enriched.ttl');
     info(`Creating asct-b: ${enrichedPath}`);
     convert(trimmedOutputPath, enrichedPath, 'ttl');
+
+    const enrichedJsonPath = resolve(obj.path, 'enriched/enriched.json');
+    convertNormalizedDataToJson(context, normalizedPath, enrichedJsonPath);
   } catch (e) {
     error(e);
   } finally {

src/enrichment/enrich-basic.js

@@ -2,13 +2,22 @@ import { existsSync, readFileSync } from 'fs';
 import { load } from 'js-yaml';
 import { resolve } from 'path';
 import { error, info } from '../utils/logging.js';
-import { cleanTemporaryFiles, convertNormalizedDataToOwl, convertNormalizedMetadataToRdf, logOutput } from './utils.js';
+import {
+  cleanTemporaryFiles,
+  convertNormalizedDataToJson,
+  convertNormalizedDataToOwl,
+  convertNormalizedMetadataToJson,
+  convertNormalizedMetadataToRdf,
+  logOutput,
+} from './utils.js';
 
 export function enrichBasicMetadata(context) {
   const { selectedDigitalObject: obj } = context;
   const normalizedPath = resolve(obj.path, 'normalized/normalized-metadata.yaml');
   const enrichedPath = resolve(obj.path, 'enriched/enriched-metadata.ttl');
   convertNormalizedMetadataToRdf(context, normalizedPath, enrichedPath, 'basic');
+  const enrichedJson = resolve(obj.path, 'enriched/enriched-metadata.json');
+  convertNormalizedMetadataToJson(context, normalizedPath, enrichedJson, 'basic');
 }
 
 export function enrichBasicData(context) {
@@ -25,6 +34,9 @@ export function enrichBasicData(context) {
 
     info('Validating digital objects in the basic...');
     validateBasic(context, digitalObjects);
+
+    const enrichedJsonPath = resolve(obj.path, 'enriched/enriched.json');
+    convertNormalizedDataToJson(context, normalizedPath, enrichedJsonPath, 'basic');
   } catch (e) {
     error(e);
   } finally {

src/enrichment/enrich-collection.js

@@ -1,21 +1,25 @@
 import { existsSync, readFileSync } from 'fs';
 import { load } from 'js-yaml';
 import { resolve } from 'path';
-import { error, header, info, more } from '../utils/logging.js';
+import { error, info, more } from '../utils/logging.js';
 import { convert, merge } from '../utils/robot.js';
 import {
   cleanTemporaryFiles,
-  convertNormalizedMetadataToRdf,
+  convertNormalizedDataToJson,
   convertNormalizedDataToOwl,
+  convertNormalizedMetadataToJson,
+  convertNormalizedMetadataToRdf,
+  logOutput,
   runCompleteClosure,
-  logOutput
 } from './utils.js';
 
 export function enrichCollectionMetadata(context) {
   const { selectedDigitalObject: obj } = context;
   const normalizedPath = resolve(obj.path, 'normalized/normalized-metadata.yaml');
   const enrichedPath = resolve(obj.path, 'enriched/enriched-metadata.ttl');
   convertNormalizedMetadataToRdf(context, normalizedPath, enrichedPath);
+  const enrichedJson = resolve(obj.path, 'enriched/enriched-metadata.json');
+  convertNormalizedMetadataToJson(context, normalizedPath, enrichedJson);
 }
 
 export function enrichCollectionData(context) {
@@ -33,7 +37,7 @@ export function enrichCollectionData(context) {
     info('Validating digital objects in the collection...');
     validateCollection(context, digitalObjects);
 
-    info('Consolidating all collection members...')
+    info('Consolidating all collection members...');
     info('Merging files:');
     const doPaths = [];
     doPaths.push(baseInputPath);
@@ -49,11 +53,13 @@ export function enrichCollectionData(context) {
     info(`Creating collection: ${enrichedPath}`);
     convert(enrichedMergePath, enrichedPath, 'ttl');
 
-    info('Optimizing collection graph...')
+    info('Optimizing collection graph...');
     const redundantPath = resolve(obj.path, 'enriched/redundant.ttl');
     runCompleteClosure(enrichedPath, redundantPath);
     logOutput(redundantPath);
 
+    const enrichedJsonPath = resolve(obj.path, 'enriched/enriched.json');
+    convertNormalizedDataToJson(context, normalizedPath, enrichedJsonPath);
   } catch (e) {
     error(e);
   } finally {