#OBS-I116: Dataset CRUD APIs test and fixes

api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json

@@ -260,10 +260,10 @@
                 },
                 "oneOf": [
                   {
-                    "required": ["denorm_dataset", "denorm_out_field", "denorm_key"]
+                    "required": ["dataset_id", "denorm_out_field", "denorm_key"]
                   },
                   {
-                    "required": ["denorm_dataset", "denorm_out_field", "jsonata_expr"]
+                    "required": ["dataset_id", "denorm_out_field", "jsonata_expr"]
                   }
                 ],
                 "additionalProperties": false

api-service/src/v2/controllers/DatasetList/DatasetList.ts

@@ -43,9 +43,9 @@ const listDatasets = async (request: Record<string, any>): Promise<Record<string
 
     const { filters = {} } = request || {};
     const datasetStatus = _.get(filters, "status");
-    const status = _.isArray(datasetStatus) ? datasetStatus : [datasetStatus]
-    const draftFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? draftDatasetStatus : _.intersection(datasetStatus, draftDatasetStatus));
-    const liveFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? liveDatasetStatus : _.intersection(datasetStatus, liveDatasetStatus));
+    const status = _.isArray(datasetStatus) ? datasetStatus : _.compact([datasetStatus])
+    const draftFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? draftDatasetStatus : _.intersection(status, draftDatasetStatus));
+    const liveFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? liveDatasetStatus : _.intersection(status, liveDatasetStatus));
     const liveDatasetList = await datasetService.findDatasets(liveFilters, defaultFields, [["updated_date", "DESC"]]);
     const draftDatasetList = await datasetService.findDraftDatasets(draftFilters, defaultFields, [["updated_date", "DESC"]]);
     return _.compact(_.concat(liveDatasetList, draftDatasetList));

api-service/src/v2/controllers/DatasetRead/DatasetRead.ts

@@ -26,7 +26,7 @@ const isValidRequest = (req: Request, res: Response): boolean => {
         } as ErrorObject, req, res);
         return false;
     }
-    const fieldValues = _.split(fields, ",")
+    const fieldValues = fields ? _.split(fields, ",") : []
     const invalidFields = _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes()))
     if (!_.isEmpty(invalidFields)) {
         logger.error({ code: "DATASET_INVALID_FIELDS", apiId, dataset_id, message: `The specified fields [${invalidFields}] in the dataset cannot be found` })
@@ -49,7 +49,7 @@ const datasetRead = async (req: Request, res: Response) => {
     }
     const { dataset_id } = req.params;
     const { fields, mode } = req.query;
-    const attributes = (fields === null) ? defaultFields : _.split(<string>fields, ",");
+    const attributes = !fields ? defaultFields : _.split(<string>fields, ",");
     const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes)
     if(!dataset) {
         logger.error({ code: "DATASET_NOT_FOUND", apiId, dataset_id, message: `Dataset with the given dataset_id:${dataset_id} not found` })

api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts

@@ -164,22 +164,44 @@ const checkDatasetDenorm = async (payload: Record<string, any>) => {
     const { dataset } = payload
     const { dataset_id, type } = dataset
     if (type === DatasetType.MasterDataset) {
-        const liveDatasets = await Dataset.findAll({ attributes: ["denorm_config"], raw: true }) || []
-        const draftDatasets = await DatasetDraft.findAll({ attributes: ["denorm_config"], raw: true }) || []
-        _.forEach([...liveDatasets, ...draftDatasets], datasets => {
-            _.forEach(_.get(datasets, "denorm_config.denorm_fields"), denorms => {
-                if (_.get(denorms, "dataset_id") === dataset_id) {
-                    logger.error(`Failed to retire dataset as it is used by other datasets:${dataset_id}`)
-                    throw {
-                        code: "DATASET_IN_USE",
-                        errCode: "BAD_REQUEST",
-                        message: "Failed to retire dataset as it is used by other datasets",
-                        statusCode: 400
-                    }
-                }
-            })
-        })
+        const liveDatasets = await Dataset.findAll({ where: { status: DatasetStatus.Live, type: DatasetType.Dataset }, attributes: ["denorm_config", "id"], raw: true }) || []
+        const draftDatasets = await DatasetDraft.findAll({ where: { status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft], type: DatasetType.Dataset }, attributes: ["denorm_config", "id"], raw: true }) || []
+        const liveDenorms = _.uniq(getDenormDatasets(liveDatasets, dataset_id))
+        const draftDenorms = _.uniq(getDenormDatasets(draftDatasets, dataset_id))
+        if (_.size([...liveDenorms, ...draftDenorms])) {
+            const denormErrMsg = getDenormErrMsg(liveDenorms, draftDenorms)
+            logger.error(denormErrMsg);
+            throw {
+                code: "DATASET_IN_USE",
+                errCode: "BAD_REQUEST",
+                message: denormErrMsg,
+                statusCode: 400
+            }
+        }
+    }
+}
+
+const getDenormErrMsg = (liveDenorms: Record<string, any>, draftDenorms: Record<string, any>) => {
+    if (_.size(liveDenorms) && _.size(draftDenorms)) {
+        return `Failed to retire dataset as it is used by Live datasets with id:[${liveDenorms}] and Draft datasets with id:[${draftDenorms}]`
+    }
+    if (_.size(liveDenorms)) {
+        return `Failed to retire dataset as it is used by Live datasets with id:[${liveDenorms}]`
     }
+    if (_.size(draftDenorms)) {
+        return `Failed to retire dataset as it is used by Draft datasets with id:[${draftDenorms}]`
+    }
+}
+
+const getDenormDatasets = (datasets: any[], dataset_id: string) => {
+    const denorms = _.map(datasets, dataset => {
+        return _.map(_.get(dataset, "denorm_config.denorm_fields"), denorms => {
+            if (_.get(denorms, "dataset_id") === dataset_id) {
+                return _.get(dataset, "id")
+            }
+        })
+    })
+    return _.compact(_.flattenDeep(denorms))
 }
 
 //SET_DATASET_TO_RETIRE

api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts

@@ -43,29 +43,15 @@ const isValidRequest = async (req: Request, res: Response): Promise<boolean> =>
         return false;
     }
 
-    const duplicateDenormKeys = datasetService.getDuplicateDenormKey(_.get(req, ["body", "request", "denorm_config"]))
-    if (!_.isEmpty(duplicateDenormKeys)) {
-        const code = "DATASET_DUPLICATE_DENORM_KEY"
-        logger.error({ code: "DATASET_DUPLICATE_DENORM_KEY", body: req.body, message: `Duplicate denorm output fields found. Duplicate Denorm out fields are [${duplicateDenormKeys}]` })
-        ResponseHandler.errorResponse({
-            code: "DATASET_DUPLICATE_DENORM_KEY",
-            statusCode: 400,
-            message: "Duplicate denorm key found",
-            errCode: "BAD_REQUEST"
-        } as ErrorObject, req, res);
-        return false;
-    }
-
     return true;
 }
 
-const isValidDataset = (datasetModel: Model<any, any> | null, req: Request, res: Response) : boolean => {
+const isValidDataset = (dataset: Record<string, any> | null, req: Request, res: Response): boolean => {
 
     const datasetId = _.get(req, ["body", "request", "dataset_id"])
     const versionKey = _.get(req, ["body", "request", "version_key"])
-    if(datasetModel) {
-        const dataset:Record<string, any> = datasetModel.toJSON
-        if(dataset.api_version !== "v2") {
+    if (dataset) {
+        if (dataset.api_version !== "v2") {
             logger.error({ code: "DATASET_API_VERSION_MISMATCH", apiId, body: req.body, message: `Draft dataset api version is not v2:${datasetId}` })
             ResponseHandler.errorResponse({
                 code: "DATASET_API_VERSION_MISMATCH",
@@ -122,15 +108,16 @@ const datasetUpdate = async (req: Request, res: Response) => {
     }
 
     const draftDataset = mergeDraftDataset(datasetModel, datasetReq);
-    const response = datasetService.updateDraftDataset(draftDataset);    
+    const response = await datasetService.updateDraftDataset(draftDataset);
     ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response });
 }
 
 const mergeDraftDataset = (datasetModel: Model<any, any> | null, datasetReq: any): Record<string, any> => {
 
-    let dataset:Record<string, any> = {
-        version_key: datasetReq.version_key,
-        name: datasetReq.name || _.get(datasetModel, ["name"])
+    let dataset: Record<string, any> = {
+        version_key: Date.now().toString(),
+        name: datasetReq.name || _.get(datasetModel, ["name"]),
+        id: _.get(datasetModel, ["id"])
     }
     if(datasetReq.validation_config) dataset["validation_config"] = datasetReq.validation_config
     if(datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config

api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json

@@ -297,7 +297,7 @@
                 "if": {
                   "properties": {
                     "action": {
-                      "enum": ["upsert", "update"]
+                      "const": "upsert"
                     }
                   }
                 },
@@ -306,10 +306,10 @@
                     "value": {
                       "oneOf": [
                         {
-                          "required": ["denorm_dataset", "denorm_out_field", "denorm_key"]
+                          "required": ["dataset_id", "denorm_out_field", "denorm_key"]
                         },
                         {
-                          "required": ["denorm_dataset", "denorm_out_field", "jsonata_expr"]
+                          "required": ["dataset_id", "denorm_out_field", "jsonata_expr"]
                         }
                       ]
                     }

api-service/src/v2/models/Dataset.ts

@@ -80,6 +80,5 @@ export const Dataset = sequelize.define("datasets", {
     tableName: "datasets",
     timestamps: true,
     createdAt: "created_date",
-    updatedAt: "updated_date",
-    paranoid: true
+    updatedAt: "updated_date"
 })

api-service/src/v2/services/DatasetService.ts

@@ -119,7 +119,8 @@ class DatasetService {
             await DatasetDraft.update(draftDataset, { where: { id: datasetId }, transaction});
             await DatasetTransformationsDraft.destroy({ where: { dataset_id: datasetId }, transaction });
             await DatasetSourceConfigDraft.destroy({ where: { dataset_id: datasetId }, transaction });
-        } catch(err) {
+            await transaction.commit();
+        } catch (err) {
             await transaction.rollback();
             throw err;
         }
@@ -140,7 +141,7 @@ class DatasetService {
 
     createDraftDatasetFromLive = async (dataset: Model<any, any>) => {
 
-        let draftDataset:any = _.omit(dataset.toJSON, ["created_date", "updated_date", "published_date"]);
+        let draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]);
         const dataset_config:any = _.get(dataset, "dataset_config");
         const api_version:any = _.get(dataset, "api_version");
         if(api_version === "v1") {
@@ -174,6 +175,7 @@ class DatasetService {
             const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "datatype", "category"]);
             draftDataset["transformations_config"] = transformations
         }
+        draftDataset["version_key"] = Date.now().toString()
         draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version
         await DatasetDraft.create(draftDataset);
         return await this.getDraftDataset(draftDataset.dataset_id);