Create collections specific for ddo versions (#225)

* Created collections for diff ddo versions.

* Created ddo collections dinamically.

* tweak.

* Changed location.

* tweak for dir location.

* Added log.

* tweak.

* removed log.

* Renamed the collections.

* removed v4 folder.

* fix review.

* refactor readJsonMappings.

* modified return statement.

schemas/op_ddo_v1.json

@@ -0,0 +1,5 @@
+{
+  "name": "op_ddo_v4.1.0",
+  "enable_nested_fields": true,
+  "fields": [{ "name": ".*", "type": "auto", "optional": true }]
+}

schemas/op_ddo_v3.json

@@ -0,0 +1,5 @@
+{
+  "name": "op_ddo_v4.3.0",
+  "enable_nested_fields": true,
+  "fields": [{ "name": ".*", "type": "auto", "optional": true }]
+}

schemas/op_ddo_v5.json

@@ -0,0 +1,5 @@
+{
+  "name": "op_ddo_v4.5.0",
+  "enable_nested_fields": true,
+  "fields": [{ "name": ".*", "type": "auto", "optional": true }]
+}

src/components/core/utils/validateDdoHandler.ts

@@ -19,7 +19,7 @@ export function getSchema(version: string = CURRENT_VERSION): string {
     CORE_LOGGER.logMessage(`Can't find schema ${version}`, true)
     return
   }
-  const path = `../../../../schemas/v4/${version}.ttl`
+  const path = `../../../../schemas/${version}.ttl`
   // Use fileURLToPath to convert the URL to a file path
   const currentModulePath = fileURLToPath(import.meta.url)
 

src/components/database/index.ts

@@ -614,11 +614,11 @@ export class Database {
         configureCustomDBTransport(this, DATABASE_LOGGER)
       }
 
-      this.ddo = await new DdoDatabase(config, schemas.ddoSchemas)
-      this.nonce = await new NonceDatabase(config, schemas.nonceSchemas)
-      this.indexer = await new IndexerDatabase(config, schemas.indexerSchemas)
-      this.logs = await new LogDatabase(config, schemas.logSchemas)
-      this.order = await new OrderDatabase(config, schemas.orderSchema)
+      this.ddo = await new DdoDatabase(this.config, schemas.ddoSchemas)
+      this.nonce = await new NonceDatabase(this.config, schemas.nonceSchemas)
+      this.indexer = await new IndexerDatabase(this.config, schemas.indexerSchemas)
+      this.logs = await new LogDatabase(this.config, schemas.logSchemas)
+      this.order = await new OrderDatabase(this.config, schemas.orderSchema)
       return this
     })() as unknown as Database
   }

src/components/database/schemas.ts

@@ -1,4 +1,48 @@
 import { TypesenseCollectionCreateSchema } from '../../@types/index.js'
+import fs from 'fs'
+import path, { dirname, resolve } from 'path'
+import { fileURLToPath } from 'url'
+import { DATABASE_LOGGER } from '../../utils/logging/common.js'
+import { LOG_LEVELS_STR } from '../../utils/logging/Logger.js'
+
+export function readJsonSchemas(): TypesenseCollectionCreateSchema[] {
+  const jsonDocuments: TypesenseCollectionCreateSchema[] = []
+  const pathToSchemaDir: string = '../../../schemas'
+  const currentModulePath = fileURLToPath(import.meta.url)
+
+  try {
+    const currentDirectory = dirname(currentModulePath)
+    const schemaFilePath = resolve(currentDirectory, pathToSchemaDir)
+    const jsonFiles = fs
+      .readdirSync(schemaFilePath)
+      .filter((file) => path.extname(file) === '.json')
+
+    if (jsonFiles.length === 0) {
+      DATABASE_LOGGER.log(
+        LOG_LEVELS_STR.LEVEL_ERROR,
+        `No JSON files found in the schemas directory ${schemaFilePath}.`,
+        true
+      )
+      return []
+    } else {
+      jsonFiles.forEach((file) => {
+        // eslint-disable-next-line security/detect-non-literal-fs-filename
+        const fileData = fs.readFileSync(path.join(schemaFilePath, file), 'utf-8')
+        const jsonFile = JSON.parse(fileData.toString())
+        jsonDocuments.push(jsonFile)
+      })
+      return jsonDocuments
+    }
+  } catch (error) {
+    DATABASE_LOGGER.log(
+      LOG_LEVELS_STR.LEVEL_ERROR,
+      `JSON mappings could not be loaded in database.
+      Error: ${error}`,
+      true
+    )
+  }
+  return []
+}
 
 export type Schema = TypesenseCollectionCreateSchema
 export type Schemas = {
@@ -8,14 +52,9 @@ export type Schemas = {
   logSchemas: Schema
   orderSchema: Schema
 }
+const ddoSchemas = readJsonSchemas()
 export const schemas: Schemas = {
-  ddoSchemas: [
-    {
-      name: 'ddo_v0.1',
-      enable_nested_fields: true,
-      fields: [{ name: '.*', type: 'auto', optional: true }] // optional: true for turning the full version DDO into short one for states DEPRECATED and REVOKED
-    }
-  ],
+  ddoSchemas,
   nonceSchemas: {
     name: 'nonce',
     enable_nested_fields: true,

src/index.ts

@@ -70,6 +70,7 @@ let dbconn = null
 if (config.dbConfig?.url) {
   // once we create a database instance, we check the environment and possibly add the DB transport
   // after that, all loggers will eventually have it too (if in production/staging environments)
+  // it creates dinamically DDO schemas
   dbconn = await new Database(config.dbConfig)
 } else {
   config.hasIndexer = false