diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index 07542074..e2bb2a53 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -1,15 +1,12 @@ import { Request, Response } from "express"; import logger from "../../logger"; -import { generateDataSource, getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService"; +import { generateDataSource, getDefaultValue, getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService"; import _ from "lodash"; import DatasetCreate from "./DatasetCreateValidationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; import { DatasetDraft } from "../../models/DatasetDraft"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; -import { defaultDatasetConfig, defaultMasterConfig } from "../../configs/DatasetConfigDefault"; -import { DatasetType } from "../../types/DatasetModels"; -import { query } from "../../connections/databaseConnection"; import { ErrorObject } from "../../types/ResponseModel"; import { DatasourceDraft } from "../../models/DatasourceDraft"; import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; @@ -102,49 +99,6 @@ const checkDatasetExists = async (dataset_id: string): Promise => { } } -const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { - const { id, dataset_id, version = 1 } = requestPayload; - const recordId = !id && `${dataset_id}.${version}` - const modifyPayload = { ...requestPayload, ...(recordId && { id: recordId }) } - const defaults = _.cloneDeep(defaultConfig) - const datasetConfigs = _.merge({ ...defaults, router_config: { topic: recordId } }, modifyPayload) - return datasetConfigs -} - -const getDatasetDefaults = async (payload: Record): Promise> => { - const datasetPayload = mergeDatasetConfigs(defaultDatasetConfig, payload) - return datasetPayload -} - -const setRedisDBConfig = async (datasetConfig: Record): Promise> => { - let nextRedisDB = datasetConfig.redis_db; - const { results }: any = await query("SELECT nextval('redis_db_index')") - if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; - return _.assign(datasetConfig, { "redis_db": nextRedisDB }) -} - -const getMasterDatasetDefaults = async (payload: Record): Promise> => { - const masterDatasetPayload = mergeDatasetConfigs(defaultMasterConfig, payload) - let datasetConfig = masterDatasetPayload.dataset_config - datasetConfig = await setRedisDBConfig(datasetConfig); - return _.assign(masterDatasetPayload, datasetConfig); -} - -const getDefaultHandler = (datasetType: string) => { - if (datasetType == DatasetType.Dataset) { - return getDatasetDefaults; - } else { - return getMasterDatasetDefaults; - } -} - -export const getDefaultValue = async (payload: Record) => { - const datasetType = _.get(payload, "type"); - const getDatasetDefaults = getDefaultHandler(datasetType) - const datasetDefaults = await getDatasetDefaults(payload) - return _.omit(datasetDefaults, ["transformations_config"]) -} - const getTransformationConfig = (configs: Record): Record => { const { transformationPayload, datasetId } = configs if (transformationPayload) { diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index 9bf4af6f..b2248e1d 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -24,10 +24,6 @@ "request": { "type": "object", "properties": { - "id": { - "type": "string", - "minLength": 1 - }, "dataset_id": { "type": "string", "minLength": 1 diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts index 2de32ead..2549b39f 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts @@ -10,10 +10,9 @@ import { DatasetDraft } from "../../models/DatasetDraft"; import logger from "../../logger"; import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { generateDataSource, getDraftTransformations, getDuplicateConfigs, setReqDatasetId } from "../../services/DatasetService"; +import { generateDataSource, getDefaultValue, getDraftTransformations, getDuplicateConfigs, setReqDatasetId } from "../../services/DatasetService"; import { DatasourceDraft } from "../../models/DatasourceDraft"; import { ingestionConfig } from "../../configs/IngestionConfig"; -import { getDefaultValue } from "../DatasetCreate/DatasetCreate"; export const apiId = "api.datasets.update"; export const invalidInputErrCode = "DATASET_UPDATE_INPUT_INVALID" diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 2668f6e0..8a8b5c81 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -7,6 +7,9 @@ import { generateIngestionSpec } from "./IngestionService"; import { ingestionConfig } from "../configs/IngestionConfig"; import { DatasetTransformations } from "../models/Transformation"; import { getUpdatedSchema } from "./DatasourceService"; +import { DatasetType } from "../types/DatasetModels"; +import { defaultDatasetConfig, defaultMasterConfig } from "../configs/DatasetConfigDefault"; +import { query } from "../connections/databaseConnection"; export const getDataset = async (datasetId: string, raw = false): Promise => { const dataset = await Dataset.findOne({ @@ -69,4 +72,47 @@ const getDataSource = (ingestionPayload: Record) => { ingestion_spec: ingestionSpec, datasource_ref: dataSource } +} + +const getDatasetDefaults = async (payload: Record): Promise> => { + const datasetPayload = mergeDatasetConfigs(defaultDatasetConfig, payload) + return datasetPayload +} + +const setRedisDBConfig = async (datasetConfig: Record): Promise> => { + let nextRedisDB = datasetConfig.redis_db; + const { results }: any = await query("SELECT nextval('redis_db_index')") + if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; + return _.assign(datasetConfig, { "redis_db": nextRedisDB }) +} + +const getMasterDatasetDefaults = async (payload: Record): Promise> => { + const masterDatasetPayload = mergeDatasetConfigs(defaultMasterConfig, payload) + let datasetConfig = masterDatasetPayload.dataset_config + datasetConfig = await setRedisDBConfig(datasetConfig); + return _.assign(masterDatasetPayload, datasetConfig); +} + +const getDefaultHandler = (datasetType: string) => { + if (datasetType == DatasetType.Dataset) { + return getDatasetDefaults; + } else { + return getMasterDatasetDefaults; + } +} + +export const getDefaultValue = async (payload: Record) => { + const datasetType = _.get(payload, "type"); + const getDatasetDefaults = getDefaultHandler(datasetType) + const datasetDefaults = await getDatasetDefaults(payload) + return _.omit(datasetDefaults, ["transformations_config"]) +} + +const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { + const { id, dataset_id } = requestPayload; + const datasetId = !id ? dataset_id : id + const modifyPayload = { ...requestPayload, id: datasetId, router_config: { topic: datasetId } } + const defaults = _.cloneDeep(defaultConfig) + const datasetConfigs = _.merge(defaults, modifyPayload) + return datasetConfigs } \ No newline at end of file diff --git a/api-service/src/v2/services/DatasourceService.ts b/api-service/src/v2/services/DatasourceService.ts index 54783862..bc9e7cf8 100644 --- a/api-service/src/v2/services/DatasourceService.ts +++ b/api-service/src/v2/services/DatasourceService.ts @@ -98,7 +98,8 @@ export const getUpdatedSchema = async (configs: Record) => { return { ...item, type: _.get(data, "_transformedFieldSchemaType") || "string", - ...data, + isModified: true, + ...data }; } return item; diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts index e5125b66..b2dbb7f3 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -386,7 +386,6 @@ export const TestInputsForDatasetCreate = { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { - "id": "sb-telemetry2", "dataset_id": "sb-ddd", "type": "master-dataset", "name": "sb-telemetry2", @@ -488,7 +487,6 @@ export const TestInputsForDatasetCreate = { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { - "id": "sb-telemetry2", "dataset_id": "sb-ddd", "type": "dataset", "name": "sb-telemetry2",