Sanketika-Obsrv/issue-tracker#234: changes from id to dataset_id with version removed

api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts

@@ -1,15 +1,12 @@
 import { Request, Response } from "express";
 import logger from "../../logger";
-import { generateDataSource, getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService";
+import { generateDataSource, getDefaultValue, getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService";
 import _ from "lodash";
 import DatasetCreate from "./DatasetCreateValidationSchema.json";
 import { schemaValidation } from "../../services/ValidationService";
 import { DatasetDraft } from "../../models/DatasetDraft";
 import { ResponseHandler } from "../../helpers/ResponseHandler";
 import httpStatus from "http-status";
-import { defaultDatasetConfig, defaultMasterConfig } from "../../configs/DatasetConfigDefault";
-import { DatasetType } from "../../types/DatasetModels";
-import { query } from "../../connections/databaseConnection";
 import { ErrorObject } from "../../types/ResponseModel";
 import { DatasourceDraft } from "../../models/DatasourceDraft";
 import { DatasetTransformationsDraft } from "../../models/TransformationDraft";
@@ -102,49 +99,6 @@ const checkDatasetExists = async (dataset_id: string): Promise<boolean> => {
     }
 }
 
-const mergeDatasetConfigs = (defaultConfig: Record<string, any>, requestPayload: Record<string, any>): Record<string, any> => {
-    const { id, dataset_id, version = 1 } = requestPayload;
-    const recordId = !id && `${dataset_id}.${version}`
-    const modifyPayload = { ...requestPayload, ...(recordId && { id: recordId }) }
-    const defaults = _.cloneDeep(defaultConfig)
-    const datasetConfigs = _.merge({ ...defaults, router_config: { topic: recordId } }, modifyPayload)
-    return datasetConfigs
-}
-
-const getDatasetDefaults = async (payload: Record<string, any>): Promise<Record<string, any>> => {
-    const datasetPayload = mergeDatasetConfigs(defaultDatasetConfig, payload)
-    return datasetPayload
-}
-
-const setRedisDBConfig = async (datasetConfig: Record<string, any>): Promise<Record<string, any>> => {
-    let nextRedisDB = datasetConfig.redis_db;
-    const { results }: any = await query("SELECT nextval('redis_db_index')")
-    if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3;
-    return _.assign(datasetConfig, { "redis_db": nextRedisDB })
-}
-
-const getMasterDatasetDefaults = async (payload: Record<string, any>): Promise<Record<string, any>> => {
-    const masterDatasetPayload = mergeDatasetConfigs(defaultMasterConfig, payload)
-    let datasetConfig = masterDatasetPayload.dataset_config
-    datasetConfig = await setRedisDBConfig(datasetConfig);
-    return _.assign(masterDatasetPayload, datasetConfig);
-}
-
-const getDefaultHandler = (datasetType: string) => {
-    if (datasetType == DatasetType.Dataset) {
-        return getDatasetDefaults;
-    } else {
-        return getMasterDatasetDefaults;
-    }
-}
-
-export const getDefaultValue = async (payload: Record<string, any>) => {
-    const datasetType = _.get(payload, "type");
-    const getDatasetDefaults = getDefaultHandler(datasetType)
-    const datasetDefaults = await getDatasetDefaults(payload)
-    return _.omit(datasetDefaults, ["transformations_config"])
-}
-
 const getTransformationConfig = (configs: Record<string, any>): Record<string, any> => {
     const { transformationPayload, datasetId } = configs
     if (transformationPayload) {

api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json

@@ -24,10 +24,6 @@
     "request": {
       "type": "object",
       "properties": {
-        "id": {
-          "type": "string",
-          "minLength": 1
-        },
         "dataset_id": {
           "type": "string",
           "minLength": 1

api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts

@@ -10,10 +10,9 @@ import { DatasetDraft } from "../../models/DatasetDraft";
 import logger from "../../logger";
 import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault";
 import { DatasetTransformationsDraft } from "../../models/TransformationDraft";
-import { generateDataSource, getDraftTransformations, getDuplicateConfigs, setReqDatasetId } from "../../services/DatasetService";
+import { generateDataSource, getDefaultValue, getDraftTransformations, getDuplicateConfigs, setReqDatasetId } from "../../services/DatasetService";
 import { DatasourceDraft } from "../../models/DatasourceDraft";
 import { ingestionConfig } from "../../configs/IngestionConfig";
-import { getDefaultValue } from "../DatasetCreate/DatasetCreate";
 
 export const apiId = "api.datasets.update";
 export const invalidInputErrCode = "DATASET_UPDATE_INPUT_INVALID"

api-service/src/v2/services/DatasetService.ts

@@ -7,6 +7,9 @@ import { generateIngestionSpec } from "./IngestionService";
 import { ingestionConfig } from "../configs/IngestionConfig";
 import { DatasetTransformations } from "../models/Transformation";
 import { getUpdatedSchema } from "./DatasourceService";
+import { DatasetType } from "../types/DatasetModels";
+import { defaultDatasetConfig, defaultMasterConfig } from "../configs/DatasetConfigDefault";
+import { query } from "../connections/databaseConnection";
 
 export const getDataset = async (datasetId: string, raw = false): Promise<any> => {
     const dataset = await Dataset.findOne({
@@ -69,4 +72,47 @@ const getDataSource = (ingestionPayload: Record<string, any>) => {
         ingestion_spec: ingestionSpec,
         datasource_ref: dataSource
     }
+}
+
+const getDatasetDefaults = async (payload: Record<string, any>): Promise<Record<string, any>> => {
+    const datasetPayload = mergeDatasetConfigs(defaultDatasetConfig, payload)
+    return datasetPayload
+}
+
+const setRedisDBConfig = async (datasetConfig: Record<string, any>): Promise<Record<string, any>> => {
+    let nextRedisDB = datasetConfig.redis_db;
+    const { results }: any = await query("SELECT nextval('redis_db_index')")
+    if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3;
+    return _.assign(datasetConfig, { "redis_db": nextRedisDB })
+}
+
+const getMasterDatasetDefaults = async (payload: Record<string, any>): Promise<Record<string, any>> => {
+    const masterDatasetPayload = mergeDatasetConfigs(defaultMasterConfig, payload)
+    let datasetConfig = masterDatasetPayload.dataset_config
+    datasetConfig = await setRedisDBConfig(datasetConfig);
+    return _.assign(masterDatasetPayload, datasetConfig);
+}
+
+const getDefaultHandler = (datasetType: string) => {
+    if (datasetType == DatasetType.Dataset) {
+        return getDatasetDefaults;
+    } else {
+        return getMasterDatasetDefaults;
+    }
+}
+
+export const getDefaultValue = async (payload: Record<string, any>) => {
+    const datasetType = _.get(payload, "type");
+    const getDatasetDefaults = getDefaultHandler(datasetType)
+    const datasetDefaults = await getDatasetDefaults(payload)
+    return _.omit(datasetDefaults, ["transformations_config"])
+}
+
+const mergeDatasetConfigs = (defaultConfig: Record<string, any>, requestPayload: Record<string, any>): Record<string, any> => {
+    const { id, dataset_id } = requestPayload;
+    const datasetId = !id ? dataset_id : id
+    const modifyPayload = { ...requestPayload, id: datasetId, router_config: { topic: datasetId } }
+    const defaults = _.cloneDeep(defaultConfig)
+    const datasetConfigs = _.merge(defaults, modifyPayload)
+    return datasetConfigs
 }

api-service/src/v2/services/DatasourceService.ts

@@ -98,7 +98,8 @@ export const getUpdatedSchema = async (configs: Record<string, any>) => {
             return {
                 ...item,
                 type: _.get(data, "_transformedFieldSchemaType") || "string",
-                ...data,
+                isModified: true,
+                ...data
             };
         }
         return item;

api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts

@@ -386,7 +386,6 @@ export const TestInputsForDatasetCreate = {
             "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d"
         },
         "request": {
-            "id": "sb-telemetry2",
             "dataset_id": "sb-ddd",
             "type": "master-dataset",
             "name": "sb-telemetry2",
@@ -488,7 +487,6 @@ export const TestInputsForDatasetCreate = {
             "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d"
         },
         "request": {
-            "id": "sb-telemetry2",
             "dataset_id": "sb-ddd",
             "type": "dataset",
             "name": "sb-telemetry2",