From 5f2face72aaf8429e05e3469158a53b3aeecb184 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 20 May 2024 13:15:05 +0530 Subject: [PATCH] Sanketika-Obsrv/issue-tracker#154: switch for generating different ingestion template --- api-service/src/configs/IngestionConfig.ts | 2 +- .../DatasetCreate/DatasetCreate.ts | 24 +------------------ api-service/src/services/DatasetService.ts | 22 +++++++++++++++++ api-service/src/services/IngestionService.ts | 14 +++++++++-- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/api-service/src/configs/IngestionConfig.ts b/api-service/src/configs/IngestionConfig.ts index 2095f601..233b9c17 100644 --- a/api-service/src/configs/IngestionConfig.ts +++ b/api-service/src/configs/IngestionConfig.ts @@ -5,7 +5,7 @@ export const ingestionConfig = { "indexCol": { "Event Arrival Time": "obsrv_meta.syncts" }, "granularitySpec": { "rollup": false, - "segmentGranularity": env.segment_granularity || "day" + "segmentGranularity": env.segment_granularity || "DAY" }, "ioconfig": { "topic": "", "taskDuration": "PT1H" }, "tuningConfig": { "maxRowPerSegment": 5000000, "taskCount": 1 }, diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index b5f0daa0..adec2c23 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -1,6 +1,6 @@ import { Request, Response } from "express"; import logger from "../../logger"; -import { getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService"; +import { generateDataSource, getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService"; import _ from "lodash"; import DatasetCreate from "./DatasetCreateValidationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; @@ -11,9 +11,7 @@ import { defaultDatasetConfig, defaultMasterConfig } from "../../configs/Dataset import { DatasetType } from "../../types/DatasetModels"; import { query, sequelize } from "../../connections/databaseConnection"; import { ErrorObject } from "../../types/ResponseModel"; -import { generateIngestionSpec } from "../../services/IngestionService"; import { DatasourceDraft } from "../../models/DatasourceDraft"; -import { ingestionConfig } from "../../configs/IngestionConfig"; import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; export const apiId = "api.datasets.create" @@ -174,24 +172,4 @@ const getTransformationConfig = (configs: Record): Record) => { - const { id } = payload - const ingestionSpec = generateIngestionSpec(payload) - const dataSource = getDataSource({ ingestionSpec, id }) - return dataSource -} - -const getDataSource = (ingestionPayload: Record) => { - const { ingestionSpec, id } = ingestionPayload - const dataSource = `${id}_${ingestionConfig.granularitySpec.segmentGranularity}` - const dataSourceId = `${id}_${dataSource}` - return { - id: dataSourceId, - datasource: dataSource, - dataset_id: id, - ingestion_spec: ingestionSpec, - datasource_ref: dataSource - } -} - export default datasetCreate; \ No newline at end of file diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 53fc4498..f19b8617 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -3,6 +3,8 @@ import _ from "lodash"; import { DatasetDraft } from "../models/DatasetDraft"; import { DatasetTransformationsDraft } from "../models/TransformationDraft"; import { Request } from "express"; +import { generateIngestionSpec } from "./IngestionService"; +import { ingestionConfig } from "../configs/IngestionConfig"; export const getDataset = async (datasetId: string): Promise => { const dataset = await Dataset.findOne({ @@ -39,4 +41,24 @@ export const setReqDatasetId = (req: Request, dataset_id: string) => { export const getDuplicateConfigs = (configs: Array) => { return _.filter(configs, (item: string, index: number) => _.indexOf(configs, item) !== index); +} + +export const generateDataSource = (payload: Record) => { + const { id } = payload + const ingestionSpec = generateIngestionSpec(payload) + const dataSource = getDataSource({ ingestionSpec, id }) + return dataSource +} + +const getDataSource = (ingestionPayload: Record) => { + const { ingestionSpec, id } = ingestionPayload + const dataSource = `${id}_${_.toLower(ingestionConfig.granularitySpec.segmentGranularity)}` + const dataSourceId = `${id}_${dataSource}` + return { + id: dataSourceId, + datasource: dataSource, + dataset_id: id, + ingestion_spec: ingestionSpec, + datasource_ref: dataSource + } } \ No newline at end of file diff --git a/api-service/src/services/IngestionService.ts b/api-service/src/services/IngestionService.ts index 3c034a7d..a4eac10f 100644 --- a/api-service/src/services/IngestionService.ts +++ b/api-service/src/services/IngestionService.ts @@ -44,10 +44,20 @@ export const generateIngestionSpec = (payload: Record) => { } const simplifiedSpec = generateExpression(_.get(data_schema, "properties"), indexCol); const generatedSpec = process(simplifiedSpec, indexCol) - const ingestionTemplate = getIngestionTemplate({ generatedSpec, id, indexCol, dataset_id }) + const ingestionTemplate = generateIngestionTemplate({ generatedSpec, id, indexCol, dataset_id, type: "druid" }) return ingestionTemplate } +const generateIngestionTemplate = (payload: Record) => { + const { type, ...rest } = payload + switch (type) { + case 'druid': + return getDruidIngestionTemplate(rest); + default: + return null; + } +} + const checkTimestampCol = (schema: Record) => { const { indexCol, data_schema } = schema if (indexCol !== defaultIndexCol) { @@ -194,7 +204,7 @@ const getObjectType = (type: string): string => { } } -export const getIngestionTemplate = (payload: Record) => { +export const getDruidIngestionTemplate = (payload: Record) => { const { id, generatedSpec, indexCol, dataset_id } = payload const { dimensions, metrics, flattenSpec } = generatedSpec const dataSource = `${id}_${ingestionConfig.granularitySpec.segmentGranularity}`