From 90cf357ba95bd6207a778ce96bf4ae8b20569a2f Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 11 Jun 2024 18:46:03 +0530 Subject: [PATCH 1/5] Sanketika-Obsrv/issue-tracker#234: changes from id to dataset_id without version --- .../src/v2/controllers/DatasetCreate/DatasetCreate.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index 07542074..14d6f8a4 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -103,11 +103,10 @@ const checkDatasetExists = async (dataset_id: string): Promise => { } const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { - const { id, dataset_id, version = 1 } = requestPayload; - const recordId = !id && `${dataset_id}.${version}` - const modifyPayload = { ...requestPayload, ...(recordId && { id: recordId }) } + const { id, dataset_id } = requestPayload; + const modifyPayload = { ...requestPayload, ...(!id && { id: dataset_id }) } const defaults = _.cloneDeep(defaultConfig) - const datasetConfigs = _.merge({ ...defaults, router_config: { topic: recordId } }, modifyPayload) + const datasetConfigs = _.merge({ ...defaults, router_config: { topic: dataset_id } }, modifyPayload) return datasetConfigs } From ca87b2d7abd2b4aa12c9650ac2a97218c94f3e18 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 12 Jun 2024 11:30:52 +0530 Subject: [PATCH 2/5] Sanketika-Obsrv/issue-tracker#234: fix: router comfig changes --- api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index 14d6f8a4..c5e1100f 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -104,9 +104,9 @@ const checkDatasetExists = async (dataset_id: string): Promise => { const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { const { id, dataset_id } = requestPayload; - const modifyPayload = { ...requestPayload, ...(!id && { id: dataset_id }) } + const modifyPayload = { ...requestPayload, ...(!id && { id: dataset_id }), router_config: { topic: id } } const defaults = _.cloneDeep(defaultConfig) - const datasetConfigs = _.merge({ ...defaults, router_config: { topic: dataset_id } }, modifyPayload) + const datasetConfigs = _.merge(defaults, modifyPayload) return datasetConfigs } From 600dcf389ebfba85ae3f81298682a496d5b7dc4a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 12 Jun 2024 15:17:43 +0530 Subject: [PATCH 3/5] Sanketika-Obsrv/issue-tracker#234: fix: transformation fields in ingestion spec --- api-service/src/v2/services/DatasourceService.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/src/v2/services/DatasourceService.ts b/api-service/src/v2/services/DatasourceService.ts index 54783862..bc9e7cf8 100644 --- a/api-service/src/v2/services/DatasourceService.ts +++ b/api-service/src/v2/services/DatasourceService.ts @@ -98,7 +98,8 @@ export const getUpdatedSchema = async (configs: Record) => { return { ...item, type: _.get(data, "_transformedFieldSchemaType") || "string", - ...data, + isModified: true, + ...data }; } return item; From fbd6f2865d260e786443f298c2d8a1821630fa1b Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 14 Jun 2024 12:34:24 +0530 Subject: [PATCH 4/5] Sanketika-Obsrv/issue-tracker#234: fix: removal of id from the dataset create schema --- .../DatasetCreate/DatasetCreate.ts | 47 +------------------ .../DatasetCreateValidationSchema.json | 4 -- .../DatasetUpdate/DatasetUpdate.ts | 3 +- api-service/src/v2/services/DatasetService.ts | 46 ++++++++++++++++++ 4 files changed, 48 insertions(+), 52 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index c5e1100f..e2bb2a53 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -1,15 +1,12 @@ import { Request, Response } from "express"; import logger from "../../logger"; -import { generateDataSource, getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService"; +import { generateDataSource, getDefaultValue, getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService"; import _ from "lodash"; import DatasetCreate from "./DatasetCreateValidationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; import { DatasetDraft } from "../../models/DatasetDraft"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; -import { defaultDatasetConfig, defaultMasterConfig } from "../../configs/DatasetConfigDefault"; -import { DatasetType } from "../../types/DatasetModels"; -import { query } from "../../connections/databaseConnection"; import { ErrorObject } from "../../types/ResponseModel"; import { DatasourceDraft } from "../../models/DatasourceDraft"; import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; @@ -102,48 +99,6 @@ const checkDatasetExists = async (dataset_id: string): Promise => { } } -const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { - const { id, dataset_id } = requestPayload; - const modifyPayload = { ...requestPayload, ...(!id && { id: dataset_id }), router_config: { topic: id } } - const defaults = _.cloneDeep(defaultConfig) - const datasetConfigs = _.merge(defaults, modifyPayload) - return datasetConfigs -} - -const getDatasetDefaults = async (payload: Record): Promise> => { - const datasetPayload = mergeDatasetConfigs(defaultDatasetConfig, payload) - return datasetPayload -} - -const setRedisDBConfig = async (datasetConfig: Record): Promise> => { - let nextRedisDB = datasetConfig.redis_db; - const { results }: any = await query("SELECT nextval('redis_db_index')") - if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; - return _.assign(datasetConfig, { "redis_db": nextRedisDB }) -} - -const getMasterDatasetDefaults = async (payload: Record): Promise> => { - const masterDatasetPayload = mergeDatasetConfigs(defaultMasterConfig, payload) - let datasetConfig = masterDatasetPayload.dataset_config - datasetConfig = await setRedisDBConfig(datasetConfig); - return _.assign(masterDatasetPayload, datasetConfig); -} - -const getDefaultHandler = (datasetType: string) => { - if (datasetType == DatasetType.Dataset) { - return getDatasetDefaults; - } else { - return getMasterDatasetDefaults; - } -} - -export const getDefaultValue = async (payload: Record) => { - const datasetType = _.get(payload, "type"); - const getDatasetDefaults = getDefaultHandler(datasetType) - const datasetDefaults = await getDatasetDefaults(payload) - return _.omit(datasetDefaults, ["transformations_config"]) -} - const getTransformationConfig = (configs: Record): Record => { const { transformationPayload, datasetId } = configs if (transformationPayload) { diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index 9bf4af6f..b2248e1d 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -24,10 +24,6 @@ "request": { "type": "object", "properties": { - "id": { - "type": "string", - "minLength": 1 - }, "dataset_id": { "type": "string", "minLength": 1 diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts index 2de32ead..2549b39f 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts @@ -10,10 +10,9 @@ import { DatasetDraft } from "../../models/DatasetDraft"; import logger from "../../logger"; import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { generateDataSource, getDraftTransformations, getDuplicateConfigs, setReqDatasetId } from "../../services/DatasetService"; +import { generateDataSource, getDefaultValue, getDraftTransformations, getDuplicateConfigs, setReqDatasetId } from "../../services/DatasetService"; import { DatasourceDraft } from "../../models/DatasourceDraft"; import { ingestionConfig } from "../../configs/IngestionConfig"; -import { getDefaultValue } from "../DatasetCreate/DatasetCreate"; export const apiId = "api.datasets.update"; export const invalidInputErrCode = "DATASET_UPDATE_INPUT_INVALID" diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 2668f6e0..8a8b5c81 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -7,6 +7,9 @@ import { generateIngestionSpec } from "./IngestionService"; import { ingestionConfig } from "../configs/IngestionConfig"; import { DatasetTransformations } from "../models/Transformation"; import { getUpdatedSchema } from "./DatasourceService"; +import { DatasetType } from "../types/DatasetModels"; +import { defaultDatasetConfig, defaultMasterConfig } from "../configs/DatasetConfigDefault"; +import { query } from "../connections/databaseConnection"; export const getDataset = async (datasetId: string, raw = false): Promise => { const dataset = await Dataset.findOne({ @@ -69,4 +72,47 @@ const getDataSource = (ingestionPayload: Record) => { ingestion_spec: ingestionSpec, datasource_ref: dataSource } +} + +const getDatasetDefaults = async (payload: Record): Promise> => { + const datasetPayload = mergeDatasetConfigs(defaultDatasetConfig, payload) + return datasetPayload +} + +const setRedisDBConfig = async (datasetConfig: Record): Promise> => { + let nextRedisDB = datasetConfig.redis_db; + const { results }: any = await query("SELECT nextval('redis_db_index')") + if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; + return _.assign(datasetConfig, { "redis_db": nextRedisDB }) +} + +const getMasterDatasetDefaults = async (payload: Record): Promise> => { + const masterDatasetPayload = mergeDatasetConfigs(defaultMasterConfig, payload) + let datasetConfig = masterDatasetPayload.dataset_config + datasetConfig = await setRedisDBConfig(datasetConfig); + return _.assign(masterDatasetPayload, datasetConfig); +} + +const getDefaultHandler = (datasetType: string) => { + if (datasetType == DatasetType.Dataset) { + return getDatasetDefaults; + } else { + return getMasterDatasetDefaults; + } +} + +export const getDefaultValue = async (payload: Record) => { + const datasetType = _.get(payload, "type"); + const getDatasetDefaults = getDefaultHandler(datasetType) + const datasetDefaults = await getDatasetDefaults(payload) + return _.omit(datasetDefaults, ["transformations_config"]) +} + +const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { + const { id, dataset_id } = requestPayload; + const datasetId = !id ? dataset_id : id + const modifyPayload = { ...requestPayload, id: datasetId, router_config: { topic: datasetId } } + const defaults = _.cloneDeep(defaultConfig) + const datasetConfigs = _.merge(defaults, modifyPayload) + return datasetConfigs } \ No newline at end of file From 0dbf95183ff19df256222f8536f4c4b255259c1c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 14 Jun 2024 12:39:10 +0530 Subject: [PATCH 5/5] Sanketika-Obsrv/issue-tracker#234: fix: test case changes --- .../src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts index e5125b66..b2dbb7f3 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -386,7 +386,6 @@ export const TestInputsForDatasetCreate = { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { - "id": "sb-telemetry2", "dataset_id": "sb-ddd", "type": "master-dataset", "name": "sb-telemetry2", @@ -488,7 +487,6 @@ export const TestInputsForDatasetCreate = { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { - "id": "sb-telemetry2", "dataset_id": "sb-ddd", "type": "dataset", "name": "sb-telemetry2",