From 6591f06a03e7ed09a7d4d5befe7988351b94d82f Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 13 Nov 2024 23:08:43 +0530 Subject: [PATCH] RC-1.2.0 (#25) * #OBS-I116: Dataset update API Dedupe and denorm test cases fixes * #OBS-I116: Dataset Create api test case fixes * #OBS-I116: Dataset update extraction config api test case fixes * #OBS-I116: Dataset update api test cases * #OBS-I116: fix: linting fixes * #OBS-I116: lint fixes * #OBS-I116: Dataset status transition test cases * #OBS-I116: feat: Test cases and linting fixes * #OBS-I116: feat: Dataset status transition test cases fix * #OBS-I141: added a new metric to sum the response time * #OBS-I141: modified the url variable and access dataset_id from params * #OBS-I141: added helper function to get dataset_id for error cases * #OBS-I141: added telemetry for v2 api's * #OBS-I141: added a new metric to sum the response time * #OBS-I141: modified the url variable and access dataset_id from params * #OBS-I141: added helper function to get dataset_id for error cases * #OBS-I141: added telemetry for v2 api's * #OBS-I141: added telemetry for v2 api's * #OBS-I143: feat: dataset publish changes to deploy flink connectors * #OBS-I141: removed metric for sum of response time * #OBS-I141: removed usage of builtin kafka methods from telemetry file * #OBS-I146: feat: Retire fix * Issue #SBCOSS-12 fix: convert all SQL raw queries to prepared statements * Issue #SBCOSS-12 fix: tags is an array, so requires empty json for null case; dataset draft deletion requires deletion of transformation and source config drafts * #SBCOSS-23: feat: dataset publish changes for redeployment * #OBS-I167 : read api changes while reading connectors according to v2 structure * #OBS-I173: fix: Ready to publish schema fix to expect connector configs as object and string * #OBS-I174: fix: Dataset read api fix to expect both v1 and v2 connectors * #OBS-I146: fix: Test case fix for read api * #OBS-I146: fix: Test case fix for read api changes * #OBS-I146: fix: status transition test cases * #OBS-I146: fix: Test case script fix * #OBS-I146: fix: Type error fix * #OBS-I141: removed metric for sum of response time * #OBS-I146: fix: Dataset read api test cases fixes * #OBS-I146: fix: Hudi spec generation test cases * #OBS-I146: fix: Test case and linting fix * merge commit * #OBS-I173: fix: Dataset web console required fixes * #OBS-I173: fix: Dataset update changes to accept type changes * #OBS-I167 : dataset read api changes to read live dataset source configs * #OBS-I146: fix: linting fix * #OBS-I146: fix: linting fix * #OBS-I167 : Added string or dict as type to connector_config * #OBS-I143: dataset publish changes fixes * #OBS-I167 : if dataset is empty return with error * #OBS-I143: inswert query fix * Issue #OBS-I144 fix: icon data as string; check default version * #OBS-143: fix: dataset publish fixes * #OBS-I167 : fix: removed duplicate code. * #OBS-I181 - Updated the event structure * #OBS-I164: added jwt token vwerification and access control to api's * #OBS-I164: added jwt token vwerification and access control to api's * #OBS-I164: modified the access roles and permissions * #OBS-I164: reading public key from env file * #OBS-I186 : fix: dataset metrics api * #OBS-I185 : fix: removed duplicate code. * #OBS-I164: modified public key variable in config * flink connector helm chart updates * flink connector helm chart updates * fix: dataset publish fixes * #OBS-I164: modified public key variable in config * #OBS-I186 : added dataset mertric api controller and route and minor change in dataset transition api * install pip requirments if applicable * #OBS-I186 : removed export statement * #OBS-I164: added config for option rbac verification * #OBS-I164: changed the middleware to rbac_middleware * #OBS-I186 : Logic moved to separate function * #OBS-I164: changed import name * #OBS-I186 : Logic moved to separate function * #OBS-I185 : fix: test case fixes * #OBS-I185 : fix: linting fix * #OBS-I164: modified config and rbac middleware * #OBS-I164: added jsonwebtoken package * V2 apis (#240) * #OBS-I115: Dataset list API refactoring * #0000: adding command api * #OBS-I116: Dataset CRUD APIs test and fixes * #OBS-I115: cmd api remove addn modules * #OBS-I116: Dataset status trasition to retire check for denorm fields * #OBS-I116: Dataset CRUD APIs test cases and fixes * #OBS-I115: Dataset Transition API refactoring * #OBS-I115: Dataset Transition API refactoring * #OBS-I116: Dataset update API Dedupe and denorm test cases fixes * #OBS-I115: Dataset Transition API refactoring and error handling refactoring * #OBS-I115: Dataset Transition API refactoring * #OBS-I115: Dataset Transition API refactoring * #OBS-I115: Remove unnecessary field fields_set * #OBS-I115: Dataset publish API - update the index of hudi spec properly for publish to handle schema evolution * #OBS-I116: Dataset create and status transition api code fix * #OBS-I116: Dataset Create api test case fixes * #OBS-I116: Dataset update extraction config api test case fixes * #OBS-I116: Dataset druid ingestion spec generation fix * #OBS-I116: express version upgraded * #OBS-I116: Dataset create api fixes * #OBS-I116: Dataset ingestion spec generation fix * #OBS-I126: updated swagger documentation * #OBS-I126: updated postman collection * #OBS-I126: updated postman collection * #OBS-I116: fix: entry topic column in datasets model * #OBS-I58 feat: Minio cloud store support - Added endpoint. as optional config to support the minio * #OBS-I126: added dataset read, list, update api's documentation and updated collection * #OBS-I116: feat: Dataschema api implementation v2 * #OBS-I116: feat: Schema validation fix * #OBS-I126: swagger doc updated * #OBS-I116: Dataset update api test cases * #OBS-I116: fix: linting fixes * #OBS-I116: lint fixes * #OBS-I116: Dataset status transition test cases * #OBS-I126 : added multiple requests example * #OBS-I126 : updated order * #OBS-I126 : updated server url * #OBS-I21: feat: dataset publish changes for connectors * #OBS-I1 updated Dataset Health API code * #OBS-I116: Dataset CRUD api fixes * #OBS-I116: fix: error codes fix * #OBS-I1 Refactored as per new changes * #OBS-I101: Update the publish API for the v2 APIs * #OBS-I101: fix db models * #OBS-I115: Remove the v1 unused API code and restructure the folders * #OBS-I1 Added DatasethealthService * #OBS-I1 Updated the imports and folders * #OBS-I2 updated dataset reset * #OBS-I116: fix: Command api and schema fixes * #OBS-I116: fix: Command api fix in db query * #OBS-I1 Added Notifications and alerts APIs * #OBS-I138: added decrypted response for the read api for connectors_config field and added defaults updated date and created date to list api * #OBS-I116: fix: feat: Dataset copy and export api implementation * #OBS-I2 Refactoring as per v2 APIs * #OBS-I2 typo fix * #OBS-I116: fix: fix: Dataset copy check for dataset fix * #OBS-I21: dataset publish changes fixes * #OBS-I116: fix: feat: Feedback fixes of removing set redis db * #OBS-I138: added cors to app * #OBS-I116: fix: feat: Dataset import api implementation * #OBS-I116: fix: fix: Dataset service fix * #OBS-I116: fix: repeated Validation method removal * #OBS-I116: fix: unused code * connector list * #OBS-I142: connector list api * #OBS-I116: fix: schema validation check for v1 exported dataset. * #OBS-I116: fix: Dataset overwrite after creation failure * #OBS-I116: fix: error handling * #OBS-I116: fix: error messages fix * #OBS-I116: fix: code fixes * #OBS-I142: formatted connector list api files * #OBS-I142: formatted connector list api files * #OBS-I142: formatted connector list file * #OBS-I138: required changes for dataset for master dataset migration from v1 to v2 * #OBS-I138: removed cors package * #OBS-I138: updated package json file * #OBS-I138: indentation fix * #OBS-I138: indentation fix * #OBS-I138: fixed indentations * #OBS-I142: updated postman collection * #OBS-I142: added connector list swagger documentation * #OBS-I142: updated postman collection * #OBS-I116: feat: Dataset Import and export api integration fixes * #OBS-I138: removed comment * #OBS-I138: throwing error if dataset is undefined * #OBS-I142: added test cases for connector list * #OBS-I138: merging dataset defaults to dataset draft record before saving * #OBS-I138: adding merged event to dataset while migrating live or draft dataset * #OBS-I142: added live_date field to defaultFields * #OBS-I145: Connector Read API * #OBS-I145: updated postman collection with connector read api * #OBS-I145: updated swagger documentation * #OBS-I145: updated the connector read api * merge changes * merge changes * Resolved merge changes * Resolved merge changes * #OBS-I145: updated the connector read api * #OBS-I138: removed datakey before merging defaults to dataset * #OBS-I145: updated postman collection and swagger documentation * #OBS-I145: added the test cases for connector read api * #OBS-I145: added a test case for connector read api * #OBS-I116: feat: Dataset import api fixes * #OBS-I116: feat: Test cases and linting fixes * #OBS-I116: feat: Dataset status transition test cases fix * #OBS-I1 updated the routes * #OBS-I116: feat: Dataset migratio method fix * #OBS-I108: feat: helm modifications for flink connectors * #OBS-I108: helm chart fixes * #OBS-I108: feat: Modify volume mounts * #OBS-I108: feat: Add PVC for JobManager * #OBS-I108: feat: change args for jobmanager command * #OBS-I141: added a new metric to sum the response time * #OBS-I141: modified the url variable and access dataset_id from params * #OBS-I141: added helper function to get dataset_id for error cases * #OBS-I108: feat: Use sidecar container to submit connector flink job * #OBS-I141: added telemetry for v2 api's * #OBS-I141: added a new metric to sum the response time * #OBS-I141: modified the url variable and access dataset_id from params * #OBS-I141: added helper function to get dataset_id for error cases * #OBS-I141: added telemetry for v2 api's * #OBS-I141: added telemetry for v2 api's * #OBS-I143: feat: dataset publish changes to deploy flink connectors * #OBS-I141: removed metric for sum of response time * #OBS-I141: removed usage of builtin kafka methods from telemetry file * #OBS-I146: feat: Retire fix * Issue #SBCOSS-12 fix: convert all SQL raw queries to prepared statements * Issue #SBCOSS-12 fix: tags is an array, so requires empty json for null case; dataset draft deletion requires deletion of transformation and source config drafts * #SBCOSS-23: feat: dataset publish changes for redeployment * #OBS-I167 : read api changes while reading connectors according to v2 structure * #OBS-I173: fix: Ready to publish schema fix to expect connector configs as object and string * #OBS-I174: fix: Dataset read api fix to expect both v1 and v2 connectors * #OBS-I146: fix: Test case fix for read api * #OBS-I146: fix: Test case fix for read api changes * #OBS-I146: fix: status transition test cases * #OBS-I146: fix: Test case script fix * #OBS-I146: fix: Type error fix * #OBS-I141: removed metric for sum of response time * #OBS-I146: fix: Dataset read api test cases fixes * #OBS-I146: fix: Hudi spec generation test cases * #OBS-I146: fix: Test case and linting fix * merge commit * #OBS-I173: fix: Dataset web console required fixes * #OBS-I173: fix: Dataset update changes to accept type changes * #OBS-I167 : dataset read api changes to read live dataset source configs * #OBS-I146: fix: linting fix * #OBS-I146: fix: linting fix * #OBS-I167 : Added string or dict as type to connector_config * #OBS-I143: dataset publish changes fixes * #OBS-I167 : if dataset is empty return with error * #OBS-I143: inswert query fix * Issue #OBS-I144 fix: icon data as string; check default version * #OBS-143: fix: dataset publish fixes * #OBS-I167 : fix: removed duplicate code. * #OBS-I181 - Updated the event structure * #OBS-I164: added jwt token vwerification and access control to api's * #OBS-I164: added jwt token vwerification and access control to api's * #OBS-I164: modified the access roles and permissions * #OBS-I164: reading public key from env file * #OBS-I186 : fix: dataset metrics api * #OBS-I185 : fix: removed duplicate code. * #OBS-I164: modified public key variable in config * flink connector helm chart updates * flink connector helm chart updates * fix: dataset publish fixes * #OBS-I164: modified public key variable in config * #OBS-I186 : added dataset mertric api controller and route and minor change in dataset transition api * install pip requirments if applicable * #OBS-I186 : removed export statement * #OBS-I164: added config for option rbac verification * #OBS-I164: changed the middleware to rbac_middleware * #OBS-I186 : Logic moved to separate function * #OBS-I164: changed import name * #OBS-I186 : Logic moved to separate function * #OBS-I185 : fix: test case fixes * #OBS-I185 : fix: linting fix * #OBS-I164: modified config and rbac middleware * #OBS-I164: added jsonwebtoken package * master url fix --------- Co-authored-by: Santhosh Vasabhaktula Co-authored-by: JeraldJF Co-authored-by: yashashk Co-authored-by: harishkumar gangula Co-authored-by: SurabhiAngadi Co-authored-by: Rakshitha-D Co-authored-by: Anand Parthasarathy Co-authored-by: Aniket Sakinala * Resolved test case issue * Merge conflicts changes * #OBS-I165: added userInfo from token to request object * #OBS-I165: updated telemetry to use user role * #OBS-I165: updated datasetCreate api to add userRole as created_by * #OBS-I165: added userRole when migrating and create table from live * #OBS-I165: added userRole for data copy api * #OBS-I165: added userRole for dataset Update * #OBS-I165: added userRole for dataset Import * #OBS-I165: added userRole for dataset status transition * #OBS-I165: modified Dataset Service to update the userRoles * #OBS-I165: added permission for queryTemplateUpdate api * #OBS-I165: added userRole for query template create and update * #OBS-I165: added userRole for alerts api * #OBS-I165: added userRole for notifications api * #OBS-I165: added userRole for silences api * #OBS-I86 : commented routes have regex * #OBS-I77 : query api changes to check datasource avialability v2 * #OBS-I77 : fix: test cases * #OBS-I165: corrections userRole access in notification * #OBS-I165: added operations_admin role and updated permissions * #OBS-I165: added rbac middleware for alert routers * #OBS-I165: modified the user permissions into a json object * #OBS-I165: removed unused code * #OBS-I203: Remove the archived and purged status transition from the API * #OBS-I165: added user permissions json * #OBS-I165: removed user roles for type * #OBS-I165: added userID to req object and importing permissions from jsonfile * #OBS-I165: handled rbac disabled scenario and updated userID instead of userRole * #OBS-I165: changed the userRole to userID * #OBS-I165: added createdby for dataset publish api * #OBS-I165: added error condition and modified status code * #OBS-I165: removed redundant code * #OBS-I222 : Resolved regexp route issue in express version 5 * #OBS-I77 : query api changes to check avialability of datasource * #OBS-I77 : fix: spell fix * #OBS-I77 : granularity spell changes * #OBS-I77 : fix: test scenarios added * #OBS-I131 : fix: stop deleting draft records if submission tasks fail * #OBS-I131 : fix: upsert if republished * #OBS-I131 : fix: test case fixes * #OBS-I165: added default userID, modified error codes, added errorhandler function * #OBS-I165: added updatedby by default when an entry created * #OBS-I165: removed redundant code * #OBS-I165: modified errorhandler * #OBS-I165: modified error message * #OBS-I227 : fix: threshold of dataset level alerts inside an array. * #OBS-I179 : feat: retire dataset related alerts and metrics on dataset retire * #OBS-I79: feat: test case fixes * #OBS-I79: feat: Alerts get fix * feat: command tag name change (#252) * feat: command tag name change (#252) (#254) * #0000 fix: ingestion spec cache fix * Ingestion spec fix (#255) * feat: command tag name change (#252) (#254) * #0000 fix: ingestion spec cache fix --------- Co-authored-by: Ravi Mula * #OBS-I230: fix: kafka command * #OBS-I230: fix: cache issues changes * #OBS-I230: fix: redis db number change on update * #OBS-I187: added model for user * #OBS-I187: added userService * #OBS-I187: modified to verify keycloak token * #OBS-I187: updated rbac middleware * #OBS-I187: add seperate function to check access * #OBS-I230: fix: integration fixes * #OBS-I230: fix: kafka command fixes (#258) * #OBS-I230: fix: kafka command fixes * #OBS-I247: fix: feedback changes. --------- Co-authored-by: Ravi Mula * #OBS-I218 update kafka connector image tags * #OBS-I288: fix: removed data_format field from connector_instance (#260) * #OBS-I285: query api fix to validate and set limit to sql queries * #I285: removed empty objects and arrays from sample data (#261) * #I285: removed empty objects and arrays from sample data * Fix code scanning alert no. 98: Loop bound injection Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> * #I285: lint issues fixed --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> * #OBS-I285: query api to parse sql query first then do regex check with postman * #OBS-I285: fix: linting issue fix * #OBS-I285: fix: lint fix * #OBS-I285: fix: Logging errors in the middleware (#263) * #OBS-I285: fix: Logging errors in the middleware * #OBS-I285: fix: Logging errors by excluding sensitive info * #OBS-I285: fix: linting issue fix * #OBS-I289 : added route (#265) * fix: Stop the connectors specific to the dataset. (#266) * fix: uninstall dataset specific spark jobs * fix: remove nested loops * fix: uninstall dataset specific spark jobs * fix: remove nested loops * #OBS-I289 fix: Fix flink connector deployments * #OBS-I289 fix: Fix Spark Connector deployments * #OBS-I307 - revert the code as this is fixed from the front end (#268) * #OBS-I330 : changed type from hudi to datalake * fix for object-store-connector cron (#269) * #OBS-I330 : fixed merging issue * #OBS-I330 : removed comment * #OBS-I330 : Using dataset v2 export api while publishing * #OBS-I330 : removed print statement * #OBS-I330 : Using dataset v2 export api while publishing (#271) * #OBS-I330 : Using dataset v2 export api while publishing * #OBS-I330 : removed print statement * #OBS-I330 : Replacing - with _ for datasource_ref and adding partition key and primary key and timestamp key in column spec of ingestion spec * Hudi fixes (#272) * #OBS-I330 : Using dataset v2 export api while publishing * #OBS-I330 : removed print statement * #OBS-I330 : Replacing - with _ for datasource_ref and adding partition key and primary key and timestamp key in column spec of ingestion spec * fix: update connector instance id when inserting to db * #OBS-I330 : removed merging defaults to keys_config * #OBS-I330 : omit merging defaults to draft dataset for keys_config * #OBS-I335: hudi spec fix * #OBS-I335: hudi spec fix * #OBS-I335: linting fix * #OBS-I334 - clear transformations on re-upload * #OBS-I334 - clear keys_config on re-upload of schema file * #OBS-I335: dataset update fix * #OBS-I335: loop bound issue fix * #OBS-I335: dataset updated as per feedbacks * append base64 prefix upon connector register * #OBS-I334 - Fix the schema update functionality * Data mapping fix (#278) * #OBS-I335: hudi spec fix (#279) --------- Co-authored-by: JeraldJF Co-authored-by: Ravi Mula Co-authored-by: Rakshitha-D Co-authored-by: SurabhiAngadi Co-authored-by: Aniket Sakinala Co-authored-by: yashashk Co-authored-by: Santhosh Vasabhaktula Co-authored-by: Anand Parthasarathy Co-authored-by: Jerald <127138957+JeraldJF@users.noreply.github.com> Co-authored-by: Rakshitha-D <115482806+Rakshitha-D@users.noreply.github.com> Co-authored-by: SurabhiAngadi <138881390+SurabhiAngadi@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> Co-authored-by: yashash <126703764+yashashkumar@users.noreply.github.com> --- .github/workflows/build_and_deploy.yaml | 2 +- api-service/package.json | 4 + ...srv API Service.postman_collection_v2.json | 1882 --------- ... => Obsrv v2 apis.postman_collection.json} | 3627 ++++++++++++++--- api-service/src/app.ts | 2 +- api-service/src/configs/Config.ts | 4 +- api-service/src/configs/IngestionConfig.ts | 29 + .../connections/commandServiceConnection.ts | 4 + api-service/src/controllers/Alerts/Alerts.ts | 16 +- api-service/src/controllers/Alerts/Silence.ts | 7 +- .../ConnectorRegisterController.ts | 121 + .../CreateTemplateController.ts | 3 + .../DataIngestion/DataIngestionController.ts | 24 +- .../controllers/DataOut/DataOutController.ts | 59 +- .../src/controllers/DataOut/QueryValidator.ts | 86 +- .../controllers/DatasetCopy/DatasetCopy.ts | 3 + .../DatasetCopy/DatasetCopyHelper.ts | 4 +- .../DatasetCreate/DatasetCreate.ts | 3 + .../DatasetCreateValidationSchema.json | 13 +- .../DatasetImport/DatasetImportHelper.ts | 3 +- .../RequestValidationSchemaV2.json | 3 + .../DatasetMetricsController.ts | 38 + .../DatasetMetricsValidationSchema.json | 12 + .../controllers/DatasetRead/DatasetRead.ts | 64 +- .../DatasetStatusTransition.ts | 120 +- .../ReadyToPublishSchema.json | 147 +- .../DatasetUpdate/DatasetUpdate.ts | 172 +- .../DatasetUpdateValidationSchema.json | 6 +- .../GenerateDataSchema/GenerateDataSchema.ts | 6 +- .../GenerateSignedURL/GenerateSignedURL.ts | 60 +- .../GenerateSignedURLValidationSchema.json | 7 + .../controllers/GenerateSignedURL/helper.ts | 64 + .../NotificationChannel/Notification.ts | 11 +- .../UpdateTemplateController.ts | 3 +- api-service/src/helpers/ResponseHandler.ts | 11 +- .../src/middlewares/RBAC_middleware.ts | 144 + .../src/middlewares/userPermissions.json | 129 + api-service/src/models/DatasetDraft.ts | 2 +- api-service/src/models/User.ts | 49 + api-service/src/routes/AlertsRouter.ts | 49 +- api-service/src/routes/DruidProxyRouter.ts | 14 +- api-service/src/routes/Router.ts | 59 +- .../src/services/CloudServices/index.ts | 3 +- api-service/src/services/DatasetService.ts | 89 +- .../DataSchemaService.ts | 60 +- .../SchemaGenerateService/SchemaMapping.json | 4 - api-service/src/services/TableGenerator.ts | 94 +- api-service/src/services/UserService.ts | 11 + api-service/src/services/managers/index.ts | 18 +- api-service/src/services/telemetry.ts | 8 +- .../DataIngestTest/DataIngestionTest.spec.ts | 61 +- .../DataOutTest/DataQueryTest.spec.ts | 122 +- .../DatasetCreate/DatasetCreate.spec.ts | 2 - .../DatasetRead/DatasetRead.spec.ts | 29 +- .../DatasetLive.spec.ts | 8 +- .../DatasetRetire.spec.ts | 10 + .../DatasetUpdate/DatasetUpdate.spec.ts | 7 +- .../DatasetUpdate/Fixtures.ts | 14 +- .../TemplateQuerying/TemplateQuerying.spec.ts | 11 + api-service/swagger-doc/openapi_v2.yml | 3 + command-service/Dockerfile | 25 +- .../flink-connector/templates/deployment.yaml | 61 +- .../helm-charts/flink-connector/values.yaml | 66 +- .../templates/_cron_release_name.tpl | 2 +- .../templates/cronjob.yaml | 3 +- .../spark-connector-cron/values.yaml | 2 +- command-service/requirements.txt | 1 + .../src/command/alert_manager_command.py | 12 +- .../src/command/command_executor.py | 5 + .../src/command/connector_command.py | 74 +- .../src/command/connector_registry.py | 123 +- .../src/command/dataset_command.py | 10 +- command-service/src/command/db_command.py | 35 +- command-service/src/command/druid_command.py | 26 +- command-service/src/command/kafka_command.py | 54 + command-service/src/config/service_config.yml | 5 +- command-service/src/model/data_models.py | 1 + command-service/src/model/db_models.py | 5 +- command-service/src/service/db_service.py | 4 +- 79 files changed, 5042 insertions(+), 3092 deletions(-) delete mode 100644 api-service/postman-collection/Obsrv API Service.postman_collection_v2.json rename api-service/postman-collection/{updated_v2_collection.json => Obsrv v2 apis.postman_collection.json} (67%) create mode 100644 api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts create mode 100644 api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts create mode 100644 api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json create mode 100644 api-service/src/controllers/GenerateSignedURL/helper.ts create mode 100644 api-service/src/middlewares/RBAC_middleware.ts create mode 100644 api-service/src/middlewares/userPermissions.json create mode 100644 api-service/src/models/User.ts create mode 100644 api-service/src/services/UserService.ts create mode 100644 command-service/src/command/kafka_command.py diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml index db26184e..a51e05bf 100644 --- a/.github/workflows/build_and_deploy.yaml +++ b/.github/workflows/build_and_deploy.yaml @@ -43,7 +43,7 @@ jobs: context: ./command-service platforms: linux/amd64 push: true - tags: ${{ vars.DOCKERHUB_USERNAME }}/flink-command-service:${{ github.ref_name }} + tags: ${{ vars.DOCKERHUB_USERNAME }}/obsrv-command-service:${{ github.ref_name }} aws-deploy: needs: [build-api-service-image, build-command-service-image] diff --git a/api-service/package.json b/api-service/package.json index 1d49c029..782cda64 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -28,11 +28,13 @@ "aws-sdk": "^2.1348.0", "axios": "^1.6.0", "body-parser": "^1.20.2", + "busboy": "^1.6.0", "compression": "^1.7.4", "dateformat": "2.0.0", "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", "http-status": "^1.5.3", + "jsonwebtoken": "^9.0.1", "kafka-node": "^5.0.0", "kafkajs": "^2.2.4", "kafkajs-snappy": "^1.1.0", @@ -59,12 +61,14 @@ "@babel/traverse": "7.23.2" }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/chai": "^4.3.3", "@types/chai-as-promised": "^7.1.5", "@types/chai-spies": "^1.0.3", "@types/compression": "^1.7.2", "@types/express": "^4.17.14", "@types/http-errors": "^2.0.1", + "@types/jsonwebtoken": "^9.0.6", "@types/kafkajs": "^1.9.0", "@types/knex": "^0.16.1", "@types/lodash": "^4.14.190", diff --git a/api-service/postman-collection/Obsrv API Service.postman_collection_v2.json b/api-service/postman-collection/Obsrv API Service.postman_collection_v2.json deleted file mode 100644 index a05836e4..00000000 --- a/api-service/postman-collection/Obsrv API Service.postman_collection_v2.json +++ /dev/null @@ -1,1882 +0,0 @@ -{ - "info": { - "_postman_id": "01a69af1-9cf2-4d45-8b11-5fb6e06ca3ef", - "name": "Obsrv API Service", - "description": "Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets.", - "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "26192103", - "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-01a69af1-9cf2-4d45-8b11-5fb6e06ca3ef?action=share&source=collection_link&creator=26192103" - }, - "item": [ - { - "name": "Dataset CRUD APIs", - "item": [ - { - "name": "Dataset create", - "protocolProfileBehavior": { - "disabledSystemHeaders": {} - }, - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n \"tag1\",\n \"tag2\"\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create", - "description": "This API allows you to create new datasets used by the analytical data source." - }, - "response": [ - { - "name": "Dataset Success", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n \"tag1\",\n \"tag2\"\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T17:56:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\":\"78hj80j9-d61e-4d4f-be78-181834eeff6d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"sb-telemetry.1\",\n \"version_key\": \"1713442037275\"\n }\n}" - }, - { - "name": "Master Dataset Success", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry-master\",\n \"type\": \"master-dataset\",\n \"name\": \"sb-telemetry-master\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": []\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T17:56:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\":\"78hj80j9-d61e-4d4f-be78-181834eeff6d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"sb-telemetry-master.1\",\n \"version_key\": \"1713442037275\"\n }\n}" - }, - { - "name": "Failure: Dataset with given dataset_id already exists", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"type\": \"master-dataset\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": []\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "Conflict", - "code": 409, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T17:56:06+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"76612ad1-241b-4ce3-8af4-88db860697f4\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset already exists\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset contains duplicate denorm out field", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry\",\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T17:59:06+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"54c5b692-dc37-432e-b556-7f396d7c9e07\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DUPLICATE_DENORM_KEY_FOUND\",\n \"message\": \"Duplicate denorm key found\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Invalid request payload provided", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\"eid\"]\n },\n \"additionalProperties\":true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T18:00:34+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"615c1e4c-8c19-44fd-b29c-c235e7cbb5f0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"SCHEMA_VALIDATION_FAILED\",\n \"message\": \"#additionalProperties should NOT have additional properties\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to database failed", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-w\",\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n \"tag1\",\n \"tag2\"\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T18:02:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"dd1c0e11-fb4c-484c-81fe-82c9e2eee053\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_CREATION_FAILURE\",\n \"message\": \"Failed to create dataset\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Dataset update", - "protocolProfileBehavior": { - "disabledSystemHeaders": {} - }, - "request": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": 1713465933736,\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"eid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update", - "description": "This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or update transformations" - }, - "response": [ - { - "name": "Updated successfully", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713465933736\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"eid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T00:16:13+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4b2c0c22-f765-46f9-80af-33d0442db5ca\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"sb-telemetry.1\",\n \"version_key\": \"1713465973004\"\n }\n}" - }, - { - "name": "Failure: Dataset does not exists to update", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"version_key\":\"78997788678\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key1\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:20:39+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"ebcf0a62-7c95-40f5-adf2-87ab90a40a80\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists to update\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Invalid payload provided", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"version_key\":\"1787838782\",\n \"name\": {},\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key1\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:22:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9408e137-ada8-48b8-99e9-90d5cdd35e35\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/name/type should be string\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: No field provided along with dataset_id", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713507455695\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:23:06+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"347f465f-f100-4095-9801-38c4943380c0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Cannot update as dataset not in draft state", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry2.1\",\n \"version_key\":\"17387881289\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key1\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:25:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a9dfc926-c893-4ec3-82b8-bc8601d928a9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE\",\n \"message\": \"Dataset cannot be updated as it is not in draft state\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset contains duplicate denorm out field", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713507462417\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"add\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key1\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:29:25+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"865b9bef-9b6d-467c-9065-6689126b0f42\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_DUPLICATE_DENORM_KEY\",\n \"message\": \"Dataset contains duplicate denorm out keys:[userdata]\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset tags to add already exists", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510056996\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:32:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e9ff0bc-d660-4c4e-a2b0-cb11246a6961\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TAGS_EXISTS\",\n \"message\": \"Dataset tags already exist\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset tags to remove do not exist", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510194427\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:33:33+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c29e7d0f-3eaf-4972-8804-c172db7b6f13\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TAGS_DO_NOT_EXIST\",\n \"message\": \"Dataset tags do not exist to remove\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset transformations to add already exists", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510277508\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"eid\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:34:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e0a5cc26-032e-4be8-81ef-c7e76867c9c1\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TRANSFORMATIONS_EXIST\",\n \"message\": \"Dataset transformations already exists\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset transformations to update do not exists", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\":\"1713510277508\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key2\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"update\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:36:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0b5f515b-1958-4597-b603-b4038d4e5846\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TRANSFORMATIONS_DO_NOT_EXIST\",\n \"message\": \"Dataset transformations do not exist to update\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset transformations to remove do not exist", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510277508\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key2\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"remove\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:37:20+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69d32411-4252-4b7f-8afa-6d436928c3d3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TRANSFORMATIONS_DO_NOT_EXIST\",\n \"message\": \"Dataset transformations do not exist to remove\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset denorm fields to add already exists", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510527662\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:38:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0b807580-a2b5-4a44-adc4-80b35b4b6529\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_DENORM_EXISTS\",\n \"message\": \"Denorm fields already exist\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset denorm to remove does not exist", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510527662\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"cdata\"\n },\n \"action\": \"remove\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:40:07+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"ce78e393-2b74-46da-a70b-2abd06ad3270\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_DENORM_DO_NOT_EXIST\",\n \"message\": \"Denorm fields do not exist to remove\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to database failed", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"17987837982\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"eid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:41:33+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"451daa85-9cb8-459a-b681-a256c6e912f0\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_FAILURE\",\n \"message\": \"Failed to update dataset\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Dataset read", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - } - ] - }, - "description": "This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through the request params. By default, the API returns the dataset of status \"Live\". \n \nValid fields that user can request are \"dataset_id,id,name,type,validation_config,extraction_config,dedup_config,data_schema,router_config,denorm_config,transformation_config,dataset_config,tags,status,version,created_by,updated_by,created_date,updated_date,published_date\"" - }, - "response": [ - { - "name": "Read Draft dataset", - "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:43:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"d02c643a-d51d-4c67-8a42-9f43e61f459e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"sb-telemetry.1\",\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"\"\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [\n \"tag3\",\n \"tag4\"\n ],\n \"status\": \"Draft\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": null,\n \"client_state\": {},\n \"version_key\": \"1713510527662\",\n \"created_date\": \"2024-04-18T23:45:00.389Z\",\n \"updated_date\": \"2024-04-19T01:38:47.670Z\",\n \"transformations_config\": [\n {\n \"field_key\": \"eid\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n }\n ]\n }\n}" - }, - { - "name": "Read Live dataset", - "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v1/datasets/read/telemetry01" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:44:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3d0efbc5-dea8-4815-a749-b9f9ae16ccfe\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry01\",\n \"dataset_id\": \"telemetry01\",\n \"type\": \"dataset\",\n \"name\": \"telemetry01\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T06:27:37.557Z\",\n \"updated_date\": \"2024-04-11T06:27:37.557Z\",\n \"transformations_config\": [\n {\n \"field_key\": \"mid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n }\n ],\n \"version\": 1\n }\n}" - }, - { - "name": "Read specific fields from the dataset", - "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft&fields=name,type,id", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - }, - { - "key": "fields", - "value": "name,type,id" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:45:44+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da752dfe-0d88-4dd6-a6a8-d858f5960f7e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"id\": \"sb-telemetry.1\"\n }\n}" - }, - { - "name": "Read version_key from the dataset", - "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft&fields=version_key", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - }, - { - "key": "fields", - "value": "version_key" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:47:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"a3ad1e17-671f-4294-a834-2ee3f255c9b3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1713510527662\"\n }\n}" - }, - { - "name": "Failure: Invalid field name provided", - "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft&fields=newname", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - }, - { - "key": "fields", - "value": "newname" - } - ] - } - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:46:03+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"302fec39-5070-4464-b02f-56b1d0418147\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset not found", - "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v1/datasets/read/new" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:46:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"cd71f8c7-6aa9-4b3a-9825-b2caab67fe8c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id not found\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to database failed", - "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v1/datasets/read/sb-telemetry.1" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:46:34+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"29d82020-101d-4506-8fb8-6f3603057064\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_READ_FAILURE\",\n \"message\": \"Failed to read dataset\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Dataset List", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\":\"master-dataset\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list", - "description": "This API allows you to list all datasets. User can apply filters on dataset status and type, sort the datasets as per requested order." - }, - "response": [ - { - "name": "Dataset list success when no filters provided", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:54:56+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e7a2c5c1-5aa9-4422-81a5-6f197d3b689e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"telemetry01\",\n \"dataset_id\": \"telemetry01\",\n \"type\": \"dataset\",\n \"name\": \"telemetry01\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T06:27:37.557Z\",\n \"updated_date\": \"2024-04-11T06:27:37.557Z\",\n \"version\": 1,\n \"transformations_config\": [\n {\n \"field_key\": \"mid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n }\n ]\n },\n {\n \"id\": \"master-telemetrry\",\n \"dataset_id\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"name\": \"master-telemetrry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T09:16:36.904Z\",\n \"updated_date\": \"2024-04-11T09:16:36.904Z\",\n \"version\": 1\n },\n {\n \"id\": \"master-telemetrry.1\",\n \"dataset_id\": \"master-telemetrry\",\n \"name\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T03:46:36.938Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"new\",\n \"name\": \"j\",\n \"skill_id\": \"aabc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 1\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T03:45:57.251Z\",\n \"updated_date\": \"2024-04-11T09:21:11.167Z\"\n },\n {\n \"id\": \"two.1\",\n \"dataset_id\": \"two\",\n \"name\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"\"\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [\n \"tag3\",\n \"tag4\"\n ],\n \"status\": \"Draft\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": null,\n \"client_state\": {},\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-15T02:22:48.270Z\",\n \"updated_date\": \"2024-04-15T02:24:03.308Z\",\n \"transformations_config\": [\n {\n \"field_key\": \"eid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n },\n {\n \"field_key\": \"cid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n }\n ]\n },\n {\n \"id\": \"telemetry01.1\",\n \"dataset_id\": \"telemetry01\",\n \"name\": \"telemetry01\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T00:57:37.581Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"old\",\n \"name\": \"j\",\n \"skill_id\": \"abc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 3\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T00:54:06.404Z\",\n \"updated_date\": \"2024-04-11T06:28:24.526Z\"\n },\n {\n \"id\": \"sb-telemetry.1\",\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"\"\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [\n \"tag3\",\n \"tag4\"\n ],\n \"status\": \"Draft\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": null,\n \"client_state\": {},\n \"version_key\": \"1713510527662\",\n \"created_date\": \"2024-04-18T23:45:00.389Z\",\n \"updated_date\": \"2024-04-19T01:38:47.670Z\",\n \"transformations_config\": [\n {\n \"field_key\": \"eid\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n }\n ]\n }\n ],\n \"count\": 6\n }\n}" - }, - { - "name": "Dataset list success when status filter provided as array", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:55:39+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"836f853f-9ad8-4567-a9bb-74c5d3bba25d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"telemetry01\",\n \"dataset_id\": \"telemetry01\",\n \"type\": \"dataset\",\n \"name\": \"telemetry01\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T06:27:37.557Z\",\n \"updated_date\": \"2024-04-11T06:27:37.557Z\",\n \"version\": 1,\n \"transformations_config\": [\n {\n \"field_key\": \"mid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n }\n ]\n },\n {\n \"id\": \"master-telemetrry\",\n \"dataset_id\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"name\": \"master-telemetrry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T09:16:36.904Z\",\n \"updated_date\": \"2024-04-11T09:16:36.904Z\",\n \"version\": 1\n }\n ],\n \"count\": 2\n }\n}" - }, - { - "name": "Dataset list success when status filter provided as string", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Publish\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:56:13+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"469ef16b-4211-4b71-b063-bbd81cc9df02\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"master-telemetrry.1\",\n \"dataset_id\": \"master-telemetrry\",\n \"name\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T03:46:36.938Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"new\",\n \"name\": \"j\",\n \"skill_id\": \"aabc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 1\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T03:45:57.251Z\",\n \"updated_date\": \"2024-04-11T09:21:11.167Z\"\n },\n {\n \"id\": \"telemetry01.1\",\n \"dataset_id\": \"telemetry01\",\n \"name\": \"telemetry01\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T00:57:37.581Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"old\",\n \"name\": \"j\",\n \"skill_id\": \"abc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 3\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T00:54:06.404Z\",\n \"updated_date\": \"2024-04-11T06:28:24.526Z\"\n }\n ],\n \"count\": 2\n }\n}" - }, - { - "name": "Dataset list success when type filter provided", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Publish\",\n \"type\": \"master-dataset\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:57:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d9aeb08-2952-485f-9944-4d118da7842e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"master-telemetrry.1\",\n \"dataset_id\": \"master-telemetrry\",\n \"name\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T03:46:36.938Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"new\",\n \"name\": \"j\",\n \"skill_id\": \"aabc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 1\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T03:45:57.251Z\",\n \"updated_date\": \"2024-04-11T09:21:11.167Z\"\n }\n ],\n \"count\": 1\n }\n}" - }, - { - "name": "Dataset list success based on sortBy values", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"master-dataset\"\n },\n \"sortBy\": [\n {\n \"field\": \"updated_date\",\n \"order\": \"asc\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"270a4e50-f7e5-4562-ae75-7a967866d065\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"master-telemetrry\",\n \"dataset_id\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"name\": \"master-telemetrry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T09:16:36.904Z\",\n \"updated_date\": \"2024-04-11T09:16:36.904Z\",\n \"version\": 1\n },\n {\n \"id\": \"master-telemetrry.1\",\n \"dataset_id\": \"master-telemetrry\",\n \"name\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T03:46:36.938Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"new\",\n \"name\": \"j\",\n \"skill_id\": \"aabc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 1\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T03:45:57.251Z\",\n \"updated_date\": \"2024-04-11T09:21:11.167Z\"\n }\n ],\n \"count\": 2\n }\n}" - }, - { - "name": "Failure: Invalid payload provided", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"f9e1fae9-4660-409d-bb4d-e29888983d4b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/additionalProperties should NOT have additional properties\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to database failed", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:59:15+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"81c3a4a5-e8ac-41de-be00-6d0300ecb106\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_FAILURE\",\n \"message\": \"Failed to list dataset\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ], - "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." - }, - { - "name": "Data Ingest", - "item": [ - { - "name": "Data ingest", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"data\": {\n \"event\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/sb-telemetry-imported" - }, - "response": [ - { - "name": "Failure: schema validation", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/test2" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "300" - }, - { - "key": "ETag", - "value": "W/\"12c-KlibM35IozkQCEPK3Z48aogSrW0\"" - }, - { - "key": "Date", - "value": "Wed, 17 Apr 2024 09:20:53 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-17T14:50:53+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"98020cf9-4239-4b6e-beea-c687d21fafa3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Data ingest success (individual)", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/test2" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "261" - }, - { - "key": "ETag", - "value": "W/\"105-27xQZBGpuJMZPC95g89XsOzED1w\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 10:15:09 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T15:45:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2de69987-4f6b-4c31-b3f1-a68018ddfabf\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" - }, - { - "name": "Data ingest successful (batch)", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/test2" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "261" - }, - { - "key": "ETag", - "value": "W/\"105-mFJEsmzd/2rti3grZtWjHTnizDo\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 10:15:54 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T15:45:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"48ff1bcf-5bd3-487d-adf8-e1066d2f9638\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" - }, - { - "name": "Dataset not found", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/added-tags" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "317" - }, - { - "key": "ETag", - "value": "W/\"13d-lTx0GQxTBAUVJyB0HvmIfDIebZg\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 10:16:31 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T15:46:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"bc678713-fcc3-40fc-8ead-22b1a80c381f\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Entry topic not found", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/sample-test" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "316" - }, - { - "key": "ETag", - "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 10:17:00 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data query", - "item": [ - { - "name": "Data query", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"granularity\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"rollups-configured\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query" - }, - "response": [ - { - "name": "Success: SQL query", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"test\",\n \"table\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/test" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "538" - }, - { - "key": "ETag", - "value": "W/\"21a-K+pNBJkYiMniehzSbVK9ufWWRLY\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:04:27 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:34:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"6b853b3a-fe62-4140-9b05-f2c6a6ba45bc\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-16T13:59:22.322Z\",\n \"school_id\": 10021,\n \"school_category\": \"secondary\",\n \"grade\": 1,\n \"gender\": \"others\",\n \"total_students\": 694,\n \"students_marked\": 694,\n \"students_present\": 137,\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null\n }\n ]\n}" - }, - { - "name": "SQL Failure: Invalid date range", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"telemetry-events\",\n \"table\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/telemetry-events" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "362" - }, - { - "key": "ETag", - "value": "W/\"16a-FIy8wK3EfIWVpWBvP7qSM0cm6v4\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:06:32 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:36:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"5aaf81d1-675d-4071-883a-f92d96656aa4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "SQL Failure: dataset not found in druid", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"taxt_trip\",\n \"table\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"tests\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/taxt_trip" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "347" - }, - { - "key": "ETag", - "value": "W/\"15b-vsW6KGK4v2u3JeTsm3XZSN6/dqk\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:07:20 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:37:20+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"14594ee4-a009-44a4-8b61-1cdf4b8f60ad\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASOURCE_NOT_FOUND\",\n \"message\": \"Datasource taxt_trip.1_day not available for querying\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "SQL Failure: Datasource not found in live table", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"undefined\",\n \"table\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/undefined" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "341" - }, - { - "key": "ETag", - "value": "W/\"155-YXF5IPOdMM4KDIobS8ujcXwYyJs\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:08:53 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:38:53+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19c4e6be-e6ec-4d0d-a027-4e5ebc23889f\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Success: native query", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\": \"test\",\n \"table\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2020-12-31/2024-01-21\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/test" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "289" - }, - { - "key": "ETag", - "value": "W/\"121-KL/17PzocISF9bLwdErxBDCblqU\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:09:42 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:39:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2e2b3931-0f07-4536-a9f1-7123e500dabb\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-16T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 2\n }\n }\n ]\n}" - }, - { - "name": "Failure: Datasource not found in live table (native)", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/telemetry-eventssss" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "351" - }, - { - "key": "ETag", - "value": "W/\"15f-gaXKko6YU3dRhaApoksdaWjOP0k\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:11:25 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:41:25+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"1ffd02b2-6123-4682-880e-354a7005c060\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Invalid date range (native)", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"telemetry-events\",\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/telemetry-events" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "362" - }, - { - "key": "ETag", - "value": "W/\"16a-sXmpPCkBQ3h2VWGFQFLg9z5TOAA\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:12:19 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:42:19+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f4bd3648-d8fd-4187-a860-3173626756d5\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset not found in druid", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"telemetry-events\",\n \"granularity\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/telemetry-events" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "348" - }, - { - "key": "ETag", - "value": "W/\"15c-9MPd/+st2IWS5BfmezwbSXOEK2Q\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:13:36 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:43:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4c5a9e26-9395-406a-95f3-2d17afe4445d\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-events not available for querying\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data exhaust", - "item": [ - { - "name": "Data exhaust", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:4444/v1/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", - "host": [ - "localhost" - ], - "port": "4444", - "path": [ - "v1", - "data", - "exhaust", - "beckn-test-data" - ], - "query": [ - { - "key": "from", - "value": "2024-02-01" - }, - { - "key": "to", - "value": "2024-02-22" - }, - { - "key": "type", - "value": "raw" - } - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Query Templates", - "item": [ - { - "name": "New Request", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "text/plain" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"queryType\": \"scan\",\n \"dataSource\": {{DATASET}},\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": {{limits}}\n}", - "options": { - "raw": { - "language": "text" - } - } - }, - "url": { - "raw": "localhost:4444/v1/template/create/json_template_2?type=json", - "host": [ - "localhost" - ], - "port": "4444", - "path": [ - "v1", - "template", - "create", - "json_template_2" - ], - "query": [ - { - "key": "type", - "value": "json" - } - ] - } - }, - "response": [] - } - ] - } - ] -} \ No newline at end of file diff --git a/api-service/postman-collection/updated_v2_collection.json b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json similarity index 67% rename from api-service/postman-collection/updated_v2_collection.json rename to api-service/postman-collection/Obsrv v2 apis.postman_collection.json index 22412089..b48c505b 100644 --- a/api-service/postman-collection/updated_v2_collection.json +++ b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json @@ -1,9 +1,11 @@ { "info": { - "_postman_id": "51471244-e9b6-453e-964d-dccbdac80e72", - "name": "V2 docs", + "_postman_id": "d3ffc748-3d71-4395-9298-e065845f7bfb", + "name": "Obsrv v2 apis", + "description": "Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets.", "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "37164806" + "_exporter_id": "26192103", + "_collection_link": "https://speeding-star-177775.postman.co/workspace/2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-d3ffc748-3d71-4395-9298-e065845f7bfb?action=share&source=collection_link&creator=26192103" }, "item": [ { @@ -21,14 +23,15 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create", + "description": "This API allows you to create new datasets used by the analytical data source." }, "response": [ { @@ -43,14 +46,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "OK", "code": 200, @@ -86,7 +89,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-15T18:44:08+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"276c042c-0f23-4b26-9b10-6fe48bbc2d3a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-15T18:44:08+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"276c042c-0f23-4b26-9b10-6fe48bbc2d3a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\"\n }\n}" }, { "name": "Success: Master dataset created successfully", @@ -100,14 +103,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "OK", "code": 200, @@ -143,7 +146,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:36:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"845076be-d9e7-4246-bb8e-07ae0ce59d1e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-master\",\n \"version_key\": \"1721099200603\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:36:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"845076be-d9e7-4246-bb8e-07ae0ce59d1e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-master\",\n \"version_key\": \"1721099200603\"\n }\n}" }, { "name": "Failure: Master dataset already exists", @@ -157,14 +160,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "Conflict", "code": 409, @@ -200,7 +203,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:37:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"138b796b-1b68-481a-a59d-1cb695c1adc9\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-master\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:37:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"138b796b-1b68-481a-a59d-1cb695c1adc9\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-master\"\n }\n}" }, { "name": "Failure: Dataset already exists", @@ -214,14 +217,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "Conflict", "code": 409, @@ -257,7 +260,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:38:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf62693c-3aa4-42ce-a5ea-4bde340740f5\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-t4\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:38:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf62693c-3aa4-42ce-a5ea-4bde340740f5\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-t4\"\n }\n}" }, { "name": "Failure: Invalid request body", @@ -271,14 +274,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "Bad Request", "code": 400, @@ -314,7 +317,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:39:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a07de860-dcbc-4ff6-822e-34b47635c8a3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:39:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a07de860-dcbc-4ff6-822e-34b47635c8a3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" }, { "name": "Success: Minimal request body", @@ -328,14 +331,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_events\",\n \"type\":\"event\", //\"master\" for master dataset\n \"name\": \"sb-telemetry\"\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_events\",\n \"type\":\"event\", //\"master\" for master dataset\n \"name\": \"sb-telemetry\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "OK", "code": 200, @@ -371,7 +374,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:14:59+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e207f4f-2be6-4a45-ab78-213bea272ae0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_events\",\n \"version_key\": \"1721133899306\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:14:59+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e207f4f-2be6-4a45-ab78-213bea272ae0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_events\",\n \"version_key\": \"1721133899306\"\n }\n}" }, { "name": "Success: Dataset created successfully with all fields", @@ -385,14 +388,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"/path/to/file1.csv\",\n \"/path/to/file2.csv\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": false,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"data_key\",\n \"partition_key\": \"partition_key\",\n \"timestamp_key\": \"time\",\n \"timestamp_format\": \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\": [\n \"tag1\"\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"/path/to/file1.csv\",\n \"/path/to/file2.csv\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": false,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"data_key\",\n \"partition_key\": \"partition_key\",\n \"timestamp_key\": \"time\",\n \"timestamp_format\": \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\": [\n \"tag1\"\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "OK", "code": 200, @@ -428,7 +431,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:19:53+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"505fb3bc-ae32-4f5b-a931-adec4d1d84ba\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t41\",\n \"version_key\": \"1721220593027\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T18:19:53+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"505fb3bc-ae32-4f5b-a931-adec4d1d84ba\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t41\",\n \"version_key\": \"1721220593027\"\n }\n}" } ] }, @@ -444,14 +447,15 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url", + "description": "This API generates presigned URLs to upload or download files from cloud" }, "response": [ { @@ -466,14 +470,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url" }, "status": "OK", "code": 200, @@ -509,7 +513,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:26:19+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5306f309-4a15-458e-89e2-29d8ac0835d4\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry_10d595.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data_33109a.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:26:19+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5306f309-4a15-458e-89e2-29d8ac0835d4\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry_10d595.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data_33109a.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject\"\n }\n ]\n}" }, { "name": "Success: Generate get url", @@ -524,14 +528,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url" }, "status": "OK", "code": 200, @@ -567,7 +571,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"009c0b2d-8acd-40b0-a807-bbacf9242771\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T09:31:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"009c0b2d-8acd-40b0-a807-bbacf9242771\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject\"\n }\n ]\n}" }, { "name": "Failure: limit exceeds", @@ -582,14 +586,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url" }, "status": "Bad Request", "code": 400, @@ -625,7 +629,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:33:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d3a606ca-47d0-4746-95a1-c8692e749959\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_URL_GENERATION_LIMIT_EXCEED\",\n \"message\": \"Pre-signed URL generation failed: limit exceeded.\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:33:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d3a606ca-47d0-4746-95a1-c8692e749959\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_URL_GENERATION_LIMIT_EXCEED\",\n \"message\": \"Pre-signed URL generation failed: limit exceeded.\",\n \"trace\": \"\"\n }\n}" }, { "name": "Failure: Invalid request", @@ -640,14 +644,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"update\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"update\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url" }, "status": "Bad Request", "code": 400, @@ -683,250 +687,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"c3e9da1c-09f3-4a3b-84ec-a19efc68b856\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_GENERATE_URL_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/access/enum must be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Data schema generator", - "request": { - "method": "POST", - "header": [ - { - "key": "Accept", - "value": "application/json, text/plain, */*", - "disabled": true - }, - { - "key": "Accept-Language", - "value": "en-GB,en", - "disabled": true - }, - { - "key": "Cache-Control", - "value": "no-store", - "disabled": true - }, - { - "key": "Connection", - "value": "keep-alive", - "disabled": true - }, - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", - "disabled": true - }, - { - "key": "Origin", - "value": "http://localhost:3001", - "disabled": true - }, - { - "key": "Pragma", - "value": "no-store", - "disabled": true - }, - { - "key": "Referer", - "value": "http://localhost:3001/console/dataset/new", - "disabled": true - }, - { - "key": "Sec-Fetch-Dest", - "value": "empty", - "disabled": true - }, - { - "key": "Sec-Fetch-Mode", - "value": "cors", - "disabled": true - }, - { - "key": "Sec-Fetch-Site", - "value": "same-origin", - "disabled": true - }, - { - "key": "Sec-GPC", - "value": "1", - "disabled": true - }, - { - "key": "User-Agent", - "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "disabled": true - }, - { - "key": "sec-ch-ua", - "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", - "disabled": true - }, - { - "key": "sec-ch-ua-mobile", - "value": "?0", - "disabled": true - }, - { - "key": "sec-ch-ua-platform", - "value": "\"macOS\"", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3007/dataset/v1/dataschema" - }, - "response": [ - { - "name": "Success: Schema generated successfully", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Accept", - "value": "application/json, text/plain, */*", - "disabled": true - }, - { - "key": "Accept-Language", - "value": "en-GB,en", - "disabled": true - }, - { - "key": "Cache-Control", - "value": "no-store", - "disabled": true - }, - { - "key": "Connection", - "value": "keep-alive", - "disabled": true - }, - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", - "disabled": true - }, - { - "key": "Origin", - "value": "http://localhost:3001", - "disabled": true - }, - { - "key": "Pragma", - "value": "no-store", - "disabled": true - }, - { - "key": "Referer", - "value": "http://localhost:3001/console/dataset/new", - "disabled": true - }, - { - "key": "Sec-Fetch-Dest", - "value": "empty", - "disabled": true - }, - { - "key": "Sec-Fetch-Mode", - "value": "cors", - "disabled": true - }, - { - "key": "Sec-Fetch-Site", - "value": "same-origin", - "disabled": true - }, - { - "key": "Sec-GPC", - "value": "1", - "disabled": true - }, - { - "key": "User-Agent", - "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "disabled": true - }, - { - "key": "sec-ch-ua", - "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", - "disabled": true - }, - { - "key": "sec-ch-ua-mobile", - "value": "?0", - "disabled": true - }, - { - "key": "sec-ch-ua-platform", - "value": "\"macOS\"", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3007/dataset/v1/dataschema" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "4414" - }, - { - "key": "ETag", - "value": "W/\"113e-ykaeY2EqBHdGqGLcr7K3WSs5fYo\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 03:28:22 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"dataset.schema.identify\",\n \"ver\": \"v1\",\n \"ts\": 1721100502574,\n \"params\": {\n \"status\": \"SUCCESS\",\n \"errmsg\": \"\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tripID' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tripID\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"VendorID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tpep_pickup_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_pickup_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_pickup_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"tpep_dropoff_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_dropoff_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_dropoff_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"passenger_count\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"trip_distance\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"RatecodeID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"store_and_fwd_flag\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"PULocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"DOLocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"payment_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primary_passenger\": {\n \"type\": \"object\",\n \"properties\": {\n \"email\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mobile\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"fare_details\": {\n \"type\": \"object\",\n \"properties\": {\n \"fare_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"extra\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mta_tax\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tip_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tolls_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"improvement_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"total_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"congestion_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 98,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T09:31:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"c3e9da1c-09f3-4a3b-84ec-a19efc68b856\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_GENERATE_URL_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/access/enum must be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" } ] }, @@ -942,14 +703,15 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721135455988\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721135455988\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update", + "description": "This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or update transformations and connectors" }, "response": [ { @@ -964,14 +726,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "OK", "code": 200, @@ -1007,7 +769,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:30:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"354f1fec-0c39-42ee-a52a-49552f847c11\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134845559\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:30:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"354f1fec-0c39-42ee-a52a-49552f847c11\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134845559\"\n }\n}" }, { "name": "Success: Updated successfully", @@ -1021,14 +783,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "OK", "code": 200, @@ -1064,7 +826,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:27:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"6d835f07-aed5-4e8b-81c2-2142cfb55c52\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:27:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"6d835f07-aed5-4e8b-81c2-2142cfb55c52\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\"\n }\n}" }, { "name": "Failure: Outdated key provided", @@ -1078,14 +840,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721064642580\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721064642580\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "Conflict", "code": 409, @@ -1121,7 +883,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:23:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fe03f6-c4c4-48f6-9d84-a32cd52f4c13\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_OUTDATED\",\n \"message\": \"The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:23:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fe03f6-c4c4-48f6-9d84-a32cd52f4c13\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_OUTDATED\",\n \"message\": \"The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates\"\n }\n}" }, { "name": "Failure: Dataset not exists to update", @@ -1135,14 +897,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "Not Found", "code": 404, @@ -1178,7 +940,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:28:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf64703c-bb6b-41bf-bc1a-c85373efd925\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists with id:telemetry_record-t41\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:28:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf64703c-bb6b-41bf-bc1a-c85373efd925\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists with id:telemetry_record-t41\"\n }\n}" }, { "name": "Failure: Invalid request", @@ -1192,14 +954,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "Bad Request", "code": 400, @@ -1235,7 +997,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:29:21+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d31672b-e5c3-4a6d-afac-d9d78011bcde\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:29:21+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d31672b-e5c3-4a6d-afac-d9d78011bcde\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" }, { "name": "Failure: No fields are provided to update", @@ -1249,14 +1011,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\"\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "Bad Request", "code": 400, @@ -1292,7 +1054,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:32:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf99b1e1-7694-4be0-ba5d-e347764736de\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:32:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf99b1e1-7694-4be0-ba5d-e347764736de\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\"\n }\n}" } ] }, @@ -1307,11 +1069,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "raw": "localhost:3000/v2/datasets/read/beckn-test-data?mode=edit", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1324,7 +1086,8 @@ "value": "edit" } ] - } + }, + "description": "This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through the request params. By default, the API returns the dataset of status \"Live\". This API accepts the parameter mode=edit to read the draft dataset. If a draft dataset is not found, it creates one using the live dataset and returns the dataset details." }, "response": [ { @@ -1337,7 +1100,7 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3007/v2/datasets/read/master-test" + "url": "localhost:3000/v2/datasets/read/master-test" }, "status": "OK", "code": 200, @@ -1373,7 +1136,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"8c8a2852-54bc-43fb-b063-7f359d11930a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"8c8a2852-54bc-43fb-b063-7f359d11930a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n }\n}" }, { "name": "Success: Read draft dataset", @@ -1386,11 +1149,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "raw": "localhost:3000/v2/datasets/read/beckn-test-data?mode=edit", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1439,7 +1202,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:41:00+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"96fd4f42-fa84-4730-bc79-d241a4e335a1\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:41:00+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"96fd4f42-fa84-4730-bc79-d241a4e335a1\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n }\n}" }, { "name": "Success: Read specific column", @@ -1452,11 +1215,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/master-test?status=Draft&fields=name,type,id", + "raw": "localhost:3000/v2/datasets/read/master-test?status=Draft&fields=name,type,id", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1509,7 +1272,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:42:16+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"02a6b03a-8bf3-4e37-8dcd-859d3e8f904e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"id\": \"master-test\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:42:16+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"02a6b03a-8bf3-4e37-8dcd-859d3e8f904e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"id\": \"master-test\"\n }\n}" }, { "name": "Success: Read version_key", @@ -1522,11 +1285,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/beckn-test-data?fields=version_key&mode=edit", + "raw": "localhost:3000/v2/datasets/read/beckn-test-data?fields=version_key&mode=edit", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1579,7 +1342,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:45:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"805e848a-d260-47c3-b55c-fc9b8323719e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1718791650227\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:45:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"805e848a-d260-47c3-b55c-fc9b8323719e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1718791650227\"\n }\n}" }, { "name": "Success: Read from draft, if not present cerate draft from live dataset and then read", @@ -1592,11 +1355,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/sample1?mode=edit", + "raw": "localhost:3000/v2/datasets/read/sample1?mode=edit", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1645,7 +1408,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:49:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da70e25b-6ad0-49a7-a39d-340d1d0c46a7\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 2,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n }\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:49:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da70e25b-6ad0-49a7-a39d-340d1d0c46a7\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 2,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n }\n }\n}" }, { "name": "Failure: Invalid field name provided", @@ -1658,11 +1421,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/telemetry_record?fields=newname", + "raw": "localhost:3000/v2/datasets/read/telemetry_record?fields=newname", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1711,7 +1474,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:50:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:50:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\"\n }\n}" }, { "name": "Failure: Dataset not found", @@ -1723,7 +1486,7 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3007/v2/datasets/read/new_telemetry_record.1" + "url": "localhost:3000/v2/datasets/read/new_telemetry_record.1" }, "status": "Not Found", "code": 404, @@ -1759,7 +1522,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:51:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"3aad3842-a76e-4fe8-b635-c7fef5f318f9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:new_telemetry_record.1 not found\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:51:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"3aad3842-a76e-4fe8-b635-c7fef5f318f9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:new_telemetry_record.1 not found\"\n }\n}" } ] }, @@ -1770,14 +1533,15 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list", + "description": "This API allows you to list all datasets. User can apply filters on dataset status and type." }, "response": [ { @@ -1787,14 +1551,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "OK", "code": 200, @@ -1830,7 +1594,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:55:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"97efe04d-e981-493d-9ee7-a6dad6887d64\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry_record-t4\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_events\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_record-master\",\n \"name\": \"sb-telemetry\",\n \"type\": \"master\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"generate-schema\",\n \"name\": \"generate-schema\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-test\",\n \"name\": \"trip-test\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample-trip-details\",\n \"name\": \"sample-trip-details\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 30\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:55:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"97efe04d-e981-493d-9ee7-a6dad6887d64\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry_record-t4\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_events\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_record-master\",\n \"name\": \"sb-telemetry\",\n \"type\": \"master\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"generate-schema\",\n \"name\": \"generate-schema\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-test\",\n \"name\": \"trip-test\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample-trip-details\",\n \"name\": \"sample-trip-details\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 30\n }\n}" }, { "name": "Success: Filter based on status as array", @@ -1839,14 +1603,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "OK", "code": 200, @@ -1882,7 +1646,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" }, { "name": "Success: Filter basen on status as string", @@ -1891,14 +1655,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"ReadyToPublish\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"ReadyToPublish\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "OK", "code": 200, @@ -1934,7 +1698,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:59:18+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a08c7ea0-bb1c-4998-b47d-a76e38e87e31\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 8\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:59:18+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a08c7ea0-bb1c-4998-b47d-a76e38e87e31\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 8\n }\n}" }, { "name": "Success: Filter based on dataset type", @@ -1943,14 +1707,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Live\",\n \"type\": \"master\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Live\",\n \"type\": \"master\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "OK", "code": 200, @@ -1986,7 +1750,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:00:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0d1ff2de-42c9-4192-b75d-84f711dbfb55\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n }\n ],\n \"count\": 2\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T18:00:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0d1ff2de-42c9-4192-b75d-84f711dbfb55\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n }\n ],\n \"count\": 2\n }\n}" }, { "name": "Failure: Invalid payload", @@ -1995,14 +1759,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "Bad Request", "code": 400, @@ -2038,7 +1802,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:02:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"add9dbe0-f362-4f99-890c-3387c998a049\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/required must have required property 'msgid'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T18:02:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"add9dbe0-f362-4f99-890c-3387c998a049\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/required must have required property 'msgid'\"\n }\n}" } ] }, @@ -2065,7 +1829,8 @@ } } }, - "url": "localhost:3007/v2/datasets/dataschema" + "url": "localhost:3000/v2/datasets/dataschema", + "description": "This api is used to generate data schema for the given dataset event." }, "response": [ { @@ -2091,7 +1856,7 @@ } } }, - "url": "localhost:3007/v2/datasets/dataschema" + "url": "localhost:3000/v2/datasets/dataschema" }, "status": "OK", "code": 200, @@ -2127,7 +1892,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:32:50+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1309aea0-9a97-46e9-bc5e-a16a8a7fb624\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'sid'. The property sid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.sid\"\n },\n {\n \"message\": \"The Property 'context.sid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.sid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.cdata[*].id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.cdata.items.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'l1'. The property l1: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.rollup.properties.l1\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uid'. The property uid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.uid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"object\": {\n \"type\": \"object\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'object'. The property object: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.object\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'tags'. The property tags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.tags\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'pageid'. The property pageid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.pageid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'subtype'. The property subtype: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.subtype\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uri'. The property uri: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.uri\"\n },\n {\n \"message\": \"The Property 'edata.uri' appears to be 'uri' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.uri\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"visits\": {\n \"type\": \"array\",\n \"items\": false,\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'visits'. The property visits: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.visits\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"level\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"message\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"params\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'syncts'. The property syncts: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.syncts\"\n },\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: '@timestamp'. The property @timestamp: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.@timestamp\"\n },\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'ex_processed'. The property ex_processed: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags.properties.ex_processed\"\n }\n ],\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'flags'. The property flags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"mid\": {\n \"path\": \"$.mid\",\n \"cardinality\": 67,\n \"index\": false\n },\n \"actor.id\": {\n \"path\": \"$.actor.properties.id\",\n \"cardinality\": 56,\n \"index\": false\n },\n \"context.sid\": {\n \"path\": \"$.context.properties.sid\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"edata.uri\": {\n \"path\": \"$.edata.properties.uri\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"context.cdata[*].id\": {\n \"path\": \"$.context.properties.cdata.items.properties.id\",\n \"cardinality\": 62,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"mid\",\n \"context.cdata[*].id\",\n \"actor.id\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-22T12:32:50+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1309aea0-9a97-46e9-bc5e-a16a8a7fb624\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'sid'. The property sid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.sid\"\n },\n {\n \"message\": \"The Property 'context.sid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.sid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.cdata[*].id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.cdata.items.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'l1'. The property l1: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.rollup.properties.l1\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uid'. The property uid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.uid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"object\": {\n \"type\": \"object\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'object'. The property object: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.object\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'tags'. The property tags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.tags\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'pageid'. The property pageid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.pageid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'subtype'. The property subtype: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.subtype\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uri'. The property uri: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.uri\"\n },\n {\n \"message\": \"The Property 'edata.uri' appears to be 'uri' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.uri\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"visits\": {\n \"type\": \"array\",\n \"items\": false,\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'visits'. The property visits: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.visits\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"level\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"message\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"params\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'syncts'. The property syncts: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.syncts\"\n },\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: '@timestamp'. The property @timestamp: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.@timestamp\"\n },\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'ex_processed'. The property ex_processed: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags.properties.ex_processed\"\n }\n ],\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'flags'. The property flags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"mid\": {\n \"path\": \"$.mid\",\n \"cardinality\": 67,\n \"index\": false\n },\n \"actor.id\": {\n \"path\": \"$.actor.properties.id\",\n \"cardinality\": 56,\n \"index\": false\n },\n \"context.sid\": {\n \"path\": \"$.context.properties.sid\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"edata.uri\": {\n \"path\": \"$.edata.properties.uri\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"context.cdata[*].id\": {\n \"path\": \"$.context.properties.cdata.items.properties.id\",\n \"cardinality\": 62,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"mid\",\n \"context.cdata[*].id\",\n \"actor.id\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" }, { "name": "Failure: Invalid request body", @@ -2152,7 +1917,7 @@ } } }, - "url": "localhost:3007/v2/datasets/dataschema" + "url": "localhost:3000/v2/datasets/dataschema" }, "status": "Bad Request", "code": 400, @@ -2188,7 +1953,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:33:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bbcc86c2-042d-4f77-bb6e-e1c9116df570\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'data'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-22T12:33:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bbcc86c2-042d-4f77-bb6e-e1c9116df570\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'data'\"\n }\n}" }, { "name": "Failure: Invalid request (config not provided)", @@ -2213,7 +1978,7 @@ } } }, - "url": "localhost:3007/v2/datasets/dataschema" + "url": "localhost:3000/v2/datasets/dataschema" }, "status": "Bad Request", "code": 400, @@ -2249,175 +2014,2857 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:35:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1f856c5e-37f0-41e9-96fb-642471228da2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'config'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-22T12:35:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1f856c5e-37f0-41e9-96fb-642471228da2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'config'\"\n }\n}" } ] - } - ] - }, - { - "name": "Connector api's", - "item": [ + }, { - "name": "Connector list", + "name": "Dataset Status Transition", "request": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry-events\",\n \"status\": \"ReadyToPublish\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/datasets/status-transition", + "description": "This API allows you to perform status transition between 2 states. Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to Live, Live to Retired and even Delete a dataset." }, "response": [ { - "name": "Failure: Invalid request body, filter option array should not be empty", + "name": "Delete success: Deleted dataset successfully", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"trip-data\",\n \"status\": \"Delete\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/datasets/status-transition" }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "399" - }, - { - "key": "ETag", - "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" - }, - { - "key": "Date", - "value": "Tue, 30 Jul 2024 09:43:14 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], + "header": [], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:18:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5948e784-37f9-4a70-85ca-86c9077ee30b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset deleted successfully\",\n \"dataset_id\": \"master.1\"\n }\n}" }, { - "name": "Success: Filtered based on status", + "name": "Delete failure: Dataset not found to delete", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"master\",\n \"status\": \"Delete\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/datasets/status-transition" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "250" - }, - { - "key": "ETag", - "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" - }, - { - "key": "Date", - "value": "Tue, 30 Jul 2024 09:55:51 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], + "header": [], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:25:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"3cdcf2af-c015-4977-9d66-364e00f1712b\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to delete\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Success: Filtered based on category", + "name": "Live success: Dataset published successfully", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record\",\n \"status\": \"Live\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/datasets/status-transition" }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:21:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"88d62970-97be-472f-9ccc-67f875d69335\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset published successfully\",\n \"dataset_id\": \"telemetry\"\n }\n}" + }, + { + "name": "Live failure: Dataset not found to publish", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry-data\",\n \"status\": \"Live\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:35:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to publish\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Live failure: Dataset in draft state", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry2\",\n \"status\": \"Live\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:37:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d56e2ed4-f008-48be-a501-164c19178419\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_NOT_READY_FOR_PUBLISH\",\n \"message\": \"Failed to publish dataset as it is in draft state\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "ReadyToPublish success: Dataset is ready to publish", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry-events\",\n \"status\": \"ReadyToPublish\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:30:04+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"84858e85-6a97-43cb-b8e9-17a7e0a43365\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to ReadyToPublish successful\",\n \"dataset_id\": \"telemetry-events\"\n }\n}" + }, + { + "name": "ReadyToPublish failure: Incomplete dataset configs", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record\",\n \"status\": \"ReadyToPublish\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:36:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"a504565b-41ff-4c0f-9d64-f96df9ed89bb\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_CONFIGS_INVALID\",\n \"message\": \"#properties/denorm_config/properties/denorm_fields/items/required must have required property 'dataset_name'\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "ReadyToPublish failure: Dataset not in draft state", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry-events\",\n \"status\": \"ReadyToPublish\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:38:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"351f5a37-87f0-47cd-bebe-e3c001256d0a\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_READYTOPUBLISH_FAILURE\",\n \"message\": \"Failed to mark dataset Ready to publish as it not in draft state\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Retire success: Dataset retired successfully", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"status\": \"Retire\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:22:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"f2285754-7d5b-4320-943d-797fb136e955\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset retired successfully\",\n \"dataset_id\": \"sb-telemetry\"\n }\n}" + }, + { + "name": "Retire Failure: Dataset not found to retire", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry2\",\n \"status\": \"Retire\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:40:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"73befbbd-60e3-48e0-9cfd-cb705dfc2b85\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to retire\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Retire Failure: Dataset is already retired", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"master-telemetrry\",\n \"status\": \"Retire\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:42:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"80208169-b1d3-41cd-816b-83fae96a4370\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_ALREADY_RETIRED\",\n \"message\": \"Dataset is already retired\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Retire failure: Cannot retire master dataset as it is used by other datasets", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry01\",\n \"status\": \"Retire\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:01:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"b88c320a-2c01-4662-a509-bd532a612c05\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_IN_USE\",\n \"message\": \"Failed to retire dataset as it is used by other datasets\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Invalid request payload provided", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry01\",\n \"status\": \"Live\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:03:56+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"ba4c86bd-b438-4582-b178-2410a5c5dd15\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_STATUS_INVALID_INPUT\",\n \"message\": \"#properties/request/properties/status/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Connection to the db failed", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry01\",\n \"status\": \"Delete\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Internal Server Error", + "code": 500, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:24:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"92928434-719f-47d4-9946-1e40ecd53253\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"error\": {\n \"code\": \"DATASET_STATUS_FAILURE\",\n \"message\": \"Failed to perform status transition on datasets\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Dataset import", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/import?override=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "override", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Dataset export", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "v1-copy" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } + }, + "response": [] + }, + { + "name": "Dataset copy", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"dataset-telemetry\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"bew-copy-live2\"\n }\n }\n}\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/copy" + }, + "response": [] + } + ], + "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." + }, + { + "name": "Connector api's", + "item": [ + { + "name": "Connector list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "response": [ + { + "name": "Failure: Invalid request body, filter option array should not be empty", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "399" + }, + { + "key": "ETag", + "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:43:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + }, + { + "name": "Success: Filtered based on status", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "250" + }, + { + "key": "ETag", + "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:55:51 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + }, + { + "name": "Success: Filtered based on category", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "2571" + }, + { + "key": "ETag", + "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:25:03 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" + }, + { + "name": "Success: Connectors list with all filter options", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4380" + }, + { + "key": "ETag", + "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:26:32 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + }, + { + "name": "Success: Connectors list without filters", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4380" + }, + { + "key": "ETag", + "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:27:37 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + } + ] + }, + { + "name": "Connector Read", + "request": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + }, + "response": [ + { + "name": "Success: Read live Connector", + "originalRequest": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "3304" + }, + { + "key": "ETag", + "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 12:47:54 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" + }, + { + "name": "Success: Read Draft Connector", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "mssql-connector-2.0.0" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "744" + }, + { + "key": "ETag", + "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" + }, + { + "key": "Date", + "value": "Thu, 01 Aug 2024 07:17:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" + }, + { + "name": "Failure: Connector not found", + "originalRequest": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "276" + }, + { + "key": "ETag", + "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" + }, + { + "key": "Date", + "value": "Thu, 01 Aug 2024 09:32:48 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data Ingest", + "item": [ + { + "name": "Data ingest", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/sb-telemetry-imported" + }, + "response": [ + { + "name": "Entry topic not found", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/sample-test" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "316" + }, + { + "key": "ETag", + "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" + }, + { + "key": "Date", + "value": "Thu, 18 Apr 2024 10:17:00 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: schema validation", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/test2" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "300" + }, + { + "key": "ETag", + "value": "W/\"12c-GNCIUUDxOZD3UfM311sjnmFIgPc\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:03:32 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:33:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"acf07609-77de-4ab5-81ea-42e41b8b95ff\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: Data ingested successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "261" + }, + { + "key": "ETag", + "value": "W/\"105-UtaCmh0qZMBRBdniNq74Gr+4YAo\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:05:48 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:35:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"9a44ac5b-ef82-46f7-92c5-c5b39ef88764\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + }, + { + "name": "Success: Data ingested successfully (batch)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n },\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/test2" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "261" + }, + { + "key": "ETag", + "value": "W/\"105-ghYxKyKmjCwnqMLgFs+qX269zs0\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:06:49 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:36:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"a220041c-0c28-415a-9687-9fb2e211f8c4\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + }, + { + "name": "Failure: Dataset not found", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/added-tags" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "317" + }, + { + "key": "ETag", + "value": "W/\"13d-a2nqq0Td6EeiGj2lxH+p9BiSz9E\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:07:29 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:37:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"d4db36b4-37b0-4170-a4cf-9d2ae8fa0416\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Entry topic not found", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/sample1" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "316" + }, + { + "key": "ETag", + "value": "W/\"13c-auYp3/ERXgtSX5SN4N/gd0cioN0\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:09:26 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:39:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"62309380-3e83-442f-877e-359ed2774bbf\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data query", + "item": [ + { + "name": "Data query", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"Select * from \\\"check\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "response": [ + { + "name": "Success: native query (interval as array type)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"dataSource\": {\n \"type\": \"table\",\n \"name\": \"test.1_rollup_month\"\n },\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "313" + }, + { + "key": "ETag", + "value": "W/\"139-De+IthAwrGNR+J11CwlNf5RSMmw\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:17:45 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:47:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"134efe35-c096-4cab-ad14-db6a8952f264\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" + }, + { + "name": "Success: native query (queryType: scan)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "313" + }, + { + "key": "ETag", + "value": "W/\"139-GxRkXDcEuYT0t/bKOD9V0bicsn0\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:21:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4fc6e3c1-4715-47b7-9137-7713fb2fbe72\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" + }, + { + "name": "Success: native query (normal intervals & queryType : timeseries)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": \"2020-12-31/2024-01-21\",\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "289" + }, + { + "key": "ETag", + "value": "W/\"121-s6cWxZurfo7P84IPd5cwnCwxEC4\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:21:49 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75d7ae48-a3db-4367-b50f-34eb99ac3480\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + }, + { + "name": "Failure: Datasource not found in live table", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/telemetry-eventssss" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "356" + }, + { + "key": "ETag", + "value": "W/\"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:13:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:43:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f922f120-2aea-49af-9a76-7312fe2eb266\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Invalid date range", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/telemetry-events" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "371" + }, + { + "key": "ETag", + "value": "W/\"173-OP8NcbSqLKFO92PIyUmMk0lNsXs\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:16:29 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:46:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"b6434700-dd92-4f64-9250-a22939e753b9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in druid", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "356" + }, + { + "key": "ETag", + "value": "W/\"164-XlGTuZDLGKbe6Alm6g23+wLUk4w\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:20:27 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:50:27+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75bd4313-d504-4fd3-92ab-ee2a685beb83\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: sql query", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": \"Select * from \\\"test\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "582" + }, + { + "key": "ETag", + "value": "W/\"246-7TUMqYT3IqZGXE69WJAfYv/tZrI\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:12:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:42:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2c981011-76da-3000-97f3-eafac939e59f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + }, + { + "name": "SQL Failure: dataset not found in druid", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "356" + }, + { + "key": "ETag", + "value": "W/\"164-/i6eur3CQhuNnbokC6iAon8EIR0\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:22:26 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:52:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19849769-c290-4599-a79e-862dcbce1124\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "SQL Failure: Datasource not found in live table", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/undefined" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "346" + }, + { + "key": "ETag", + "value": "W/\"15a-pPLEc13PzXoPsR8+egdEub66a5g\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:23:02 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:53:02+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"530a12f9-04f3-4986-8f52-b3184b9194bd\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data exhaust", + "item": [ + { + "name": "Data exhaust", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "exhaust", + "beckn-test-data" + ], + "query": [ + { + "key": "from", + "value": "2024-02-01" + }, + { + "key": "to", + "value": "2024-02-22" + }, + { + "key": "type", + "value": "raw" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Query Templates", + "item": [ + { + "name": "Read template", + "request": { + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/sql_template_11" + }, + "response": [ + { + "name": "Success: JSON template", + "originalRequest": { + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/jsontemplate111" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "537" + }, + { + "key": "ETag", + "value": "W/\"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:24:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ad1c0a98-1ba3-4203-9fac-59ccaf442347\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n }\n}" + }, + { + "name": "Success: SQL template", + "originalRequest": { + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/sql1" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "528" + }, + { + "key": "ETag", + "value": "W/\"210-ar2vOqjihmm1SMoqGKcF1ROOKkg\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:24:38 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"70047866-a4b4-4fce-b9c4-4699fcab2dc6\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-05-13T07:29:04.117Z\",\n \"updated_date\": \"2024-05-13T07:29:04.117Z\"\n }\n}" + }, + { + "name": "Failure : Template not exists", + "originalRequest": { + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/sql100" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "294" + }, + { + "key": "ETag", + "value": "W/\"126-zq5j1yor+xr1XFEEbP09fQTDI/k\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:25:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:55:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"958bf0e7-bdb8-4153-abdc-ab84e8004a0e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template sql100 does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Delete template", + "request": { + "method": "DELETE", + "header": [], + "url": "localhost:3000/v2/template/delete/yashash-k" + }, + "response": [ + { + "name": "Success: Deleted successfully", + "originalRequest": { + "method": "DELETE", + "header": [], + "url": "localhost:3000/v2/template/delete/yashash-k" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "241" + }, + { + "key": "ETag", + "value": "W/\"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:28:27 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3e2859df-f494-4c47-ae64-e2c34f4ef1cb\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Template yashash-k deleted successfully\"\n }\n}" + }, + { + "name": "Failure: Template does not exists", + "originalRequest": { + "method": "DELETE", + "header": [], + "url": "localhost:3000/v2/template/delete/json_template" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "303" + }, + { + "key": "ETag", + "value": "W/\"12f-99pWw8VTwuVfDAhinC55JXfNyyg\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:28:41 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:58:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"9b7f81fb-6705-4d32-9bd3-139cd5a211b9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template json_template does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Create query template", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"query_type\": \"sql\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create/json_template_111" + }, + "response": [ + { + "name": "Success : Create query template success (json)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"json11template\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "359" + }, + { + "key": "ETag", + "value": "W/\"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 16:59:01 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"71372ce0-16b9-4594-8db8-f12eff7e6a42\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + }, + { + "name": "Success: Create query template success (SQL)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"sql1_new\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT COUNT(*) FROM \\\"{{DATASET}}\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "339" + }, + { + "key": "ETag", + "value": "W/\"153-+bOl9+lLq4m+Uttv+Euq8i/zBzE\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 16:59:58 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62aa8c6d-c49c-41c2-9cc8-2f1b02bc2388\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + }, + { + "name": "Failure: Template already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaa\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "350" + }, + { + "key": "ETag", + "value": "W/\"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:03:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:33:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"18b6b123-4df5-4124-b6ec-73b667250e1c\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_ALREADY_EXISTS\",\n \"message\": \"Template josnaks-aaa already exists\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Missing required variables", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaaq\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "407" + }, + { + "key": "ETag", + "value": "W/\"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:05:10 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:35:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62e18342-7e25-4122-8fca-6fb12fac3ff0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID\",\n \"message\": \"Invalid template provided, A template should consist of variables undefined and type of json,sql\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Schema validation failure", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"test\",\n \"query_type\": \"sq\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "416" + }, + { + "key": "ETag", + "value": "W/\"1a0-au5PdMUOZbCe2RXYjw+SJZ1EwLs\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:13:57 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:43:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/oneOf/0/properties/query_type/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "List templates", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n },\n \"sortBy\": [\n {\n \"field\": \"created_date\",\n \"order\": \"desc\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "response": [ + { + "name": "empty request body", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "6864" + }, + { + "key": "ETag", + "value": "W/\"1ad0-xp24UiXXXiFWplmv5Acja7prSYM\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 13:46:03 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T19:16:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5d41ee6e-2fc6-4353-b6c4-49e068f39b2f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:30:59.381Z\",\n \"updated_date\": \"2024-04-29T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:31:53.976Z\",\n \"updated_date\": \"2024-04-29T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n }\n ]\n}" + }, + { + "name": "Success : order by", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"order\": [\n [\n \"created_date\",\n \"ASC\"\n ]\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "7700" + }, + { + "key": "ETag", + "value": "W/\"1e14-LtQGbX2UhHK2p4juFd3eddoFjQI\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:06:09 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:36:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4393ac57-d441-4be8-b22b-9e4328cab887\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"timeseries\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"aggregations\\\":[{\\\"type\\\":\\\"filtered\\\",\\\"aggregator\\\":{\\\"type\\\":\\\"count\\\",\\\"name\\\":\\\"a0\\\"},\\\"filter\\\":{\\\"type\\\":\\\"not\\\",\\\"field\\\":{\\\"type\\\":\\\"null\\\",\\\"column\\\":\\\"school_id\\\"}},\\\"name\\\":\\\"school_id\\\"}]}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:01.096Z\",\n \"updated_date\": \"2024-04-29T11:29:01.096Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + }, + { + "name": "Success : Filters", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "2708" + }, + { + "key": "ETag", + "value": "W/\"a94-N1DxAfP8gWdksidnCf3y626Dg3s\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:08:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69da1ef2-c2c5-4f22-bb68-abdf823f0744\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + }, + { + "name": "Success : limit and offset", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"limit\":5,\n \"offset\":0\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1921" + }, + { + "key": "ETag", + "value": "W/\"781-lpOJppPfhjlnrbwoZ6q194w28Xs\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:11:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:41:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d1aa35c3-e817-4e2e-85f5-dfd346122192\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n }\n ]\n}" + } + ] + }, + { + "name": "update template", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "response": [ + { + "name": "Success: update successful", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "314" + }, + { + "key": "ETag", + "value": "W/\"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:51:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:21:47+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e4a6959-0eb9-4fc4-8e6f-2eea534d1384\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Query template updated successfully\",\n \"templateId\": \"sql11template1\"\n }\n}" + }, + { + "name": "Failure: required variables not exists to update", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "429" + }, + { + "key": "ETag", + "value": "W/\"1ad-5sb8WUekFL8s4c1Ink6bUByoHho\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:53:54 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:23:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"66b95cb3-2ef2-4735-9045-2674da552dbd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Invalid template provided, A template should consist of variables DATASET,STARTDATE,ENDDATE and type of json,sql\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Template name validation failure", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_. template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "405" + }, + { + "key": "ETag", + "value": "W/\"195-Pvg0z+WwBBq8XA4W0J0QalaIepY\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:56:59 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:26:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d6428fcf-53c9-465d-9431-769218f775b8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Template name should contain alphanumeric characters and single space between characters\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: query_type should when updating query", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_. template\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", - "value": "2571" + "value": "412" }, { "key": "ETag", - "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" + "value": "W/\"19c-ETwoh4/x7I2s9qIcbYDNmUId4XQ\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:25:03 GMT" + "value": "Fri, 10 May 2024 07:02:57 GMT" }, { "key": "Connection", @@ -2429,26 +4876,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:32:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c7a8675a-73f2-4764-abba-bfdf9f8b4621\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query_type when property query is present\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Success: Connectors list with all filter options", + "name": "Failure: query should present when updating query_type", "originalRequest": { - "method": "POST", + "method": "PATCH", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"json\"\n // \"query\": \"\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/template/update/sql11template1" }, - "status": "OK", - "code": 200, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -2461,15 +4908,15 @@ }, { "key": "Content-Length", - "value": "4380" + "value": "412" }, { "key": "ETag", - "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" + "value": "W/\"19c-nYBbts9obeaddDUgxH60sH5LgFQ\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:26:32 GMT" + "value": "Fri, 10 May 2024 07:04:55 GMT" }, { "key": "Connection", @@ -2481,26 +4928,122 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:34:55+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c1098b2-d7b3-4d39-98ee-e3e790fd23b4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query when property query_type is present\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "query template", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": "localhost:3000/v2/template/query/sql_test_1" + }, + "response": [ { - "name": "Success: Connectors list without filters", + "name": "Failure: invalid date range (native template)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/telemetry-events", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "telemetry-events" + ], + "query": [ + { + "key": "limit", + "value": "1", + "disabled": true + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "336" + }, + { + "key": "ETag", + "value": "W/\"150-T/XeSIt7PR7GcGEbET1e8n9zX7k\"" + }, + { + "key": "Date", + "value": "Thu, 02 May 2024 07:29:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-02T12:59:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"4379e16b-2fa3-46a8-8ded-bc53f56283e9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in druid", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/template/query/sql_test_1" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -2513,15 +5056,15 @@ }, { "key": "Content-Length", - "value": "4380" + "value": "357" }, { "key": "ETag", - "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" + "value": "W/\"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:27:37 GMT" + "value": "Mon, 13 May 2024 07:51:46 GMT" }, { "key": "Connection", @@ -2533,27 +5076,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" - } - ] - }, - { - "name": "Connector Read", - "request": { - "method": "GET", - "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" - }, - "response": [ + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:21:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b35a7050-b94c-4944-9630-233c9542272e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table hour is not available for querying\",\n \"trace\": \"\"\n }\n}" + }, { - "name": "Success: Read live Connector", + "name": "Failure: Datasource not found in live table", "originalRequest": { - "method": "GET", + "method": "POST", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test11\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -2566,15 +5108,15 @@ }, { "key": "Content-Length", - "value": "3304" + "value": "343" }, { "key": "ETag", - "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" + "value": "W/\"157-RFSDKlFaIxNLaNkfCfKqsUcLowk\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 12:47:54 GMT" + "value": "Mon, 13 May 2024 07:53:47 GMT" }, { "key": "Connection", @@ -2586,33 +5128,81 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:23:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"3a303dfd-1d95-4788-b1a7-d88809d4dcf3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource test11 not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Success: Read Draft Connector", + "name": "Failure: invalid date range", "originalRequest": { - "method": "GET", + "method": "POST", "header": [], - "url": { - "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "connectors", - "read", - "mssql-connector-2.0.0" - ], - "query": [ - { - "key": "mode", - "value": "edit" + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" } - ] + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "371" + }, + { + "key": "ETag", + "value": "W/\"173-5xMwCPLPBFPXCvzFzMzFY8weqGs\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:58:18 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:28:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"20391fb8-2be8-48b5-a16f-fca150580e97\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: JSON template with request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/jsontemplate1" }, "status": "OK", "code": 200, @@ -2628,15 +5218,15 @@ }, { "key": "Content-Length", - "value": "744" + "value": "301" }, { "key": "ETag", - "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" + "value": "W/\"12d-9hKB38iHEwYPT2MgF8puXcq05Ew\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 07:17:12 GMT" + "value": "Tue, 14 May 2024 06:22:24 GMT" }, { "key": "Connection", @@ -2648,17 +5238,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:52:24+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-01T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" }, { - "name": "Failure: Connector not found", + "name": "Success: SQL template query with request body", "originalRequest": { - "method": "GET", + "method": "POST", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"week\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2671,15 +5270,15 @@ }, { "key": "Content-Length", - "value": "276" + "value": "594" }, { "key": "ETag", - "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" + "value": "W/\"252-GyZnr24ylWKlH/bS4kmbMRx5jes\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 09:32:48 GMT" + "value": "Tue, 14 May 2024 06:03:06 GMT" }, { "key": "Connection", @@ -2691,7 +5290,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:33:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"48c194ee-6e73-4ee7-83e6-8b154e441911\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" } ] } diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 962331e9..6615ad74 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -22,7 +22,7 @@ app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); app.use("/", metricRouter); -app.use("*", ResponseHandler.routeNotFound); +app.use(/(.*)/, ResponseHandler.routeNotFound); app.use(obsrvErrorHandler); app.listen(config.api_port, () => { diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 38fe624c..b26a6906 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -112,5 +112,7 @@ export const config = { "dialect": process.env.dialet || "postgres", "url": process.env.grafana_url || "http://localhost:8000", "access_token": process.env.grafana_token || "" - } + }, + "user_token_public_key": process.env.user_token_public_key || "", + "is_RBAC_enabled": process.env.is_rbac_enabled || "false", } diff --git a/api-service/src/configs/IngestionConfig.ts b/api-service/src/configs/IngestionConfig.ts index 8826b264..cfc4d1cf 100644 --- a/api-service/src/configs/IngestionConfig.ts +++ b/api-service/src/configs/IngestionConfig.ts @@ -54,6 +54,13 @@ export const rawIngestionSpecDefaults = { "type": "text", "expr": "$.obsrv_meta.syncts" }, + "hudiSynctsField": { + "name": "obsrv_meta_syncts", + "arrival_format": "text", + "data_type": "date", + "type": "text", + "expr": "$.obsrv_meta.syncts" + }, "dimensions": [ { "type": "string", @@ -64,6 +71,16 @@ export const rawIngestionSpecDefaults = { "name": "obsrv.meta.source.id" } ], + "hudi_dimensions": [ + { + "type": "string", + "name": "obsrv_meta_source_connector" + }, + { + "type": "string", + "name": "obsrv_meta_source_id" + } + ], "flattenSpec": [ { "type": "path", @@ -75,5 +92,17 @@ export const rawIngestionSpecDefaults = { "expr": "$.obsrv_meta.source.connectorInstance", "name": "obsrv.meta.source.id" } + ], + "hudi_flattenSpec": [ + { + "type": "path", + "expr": "$.obsrv_meta.source.connector", + "name": "obsrv_meta_source_connector" + }, + { + "type": "path", + "expr": "$.obsrv_meta.source.connectorInstance", + "name": "obsrv_meta_source_id" + } ] } \ No newline at end of file diff --git a/api-service/src/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts index 94597474..fc77fc77 100644 --- a/api-service/src/connections/commandServiceConnection.ts +++ b/api-service/src/connections/commandServiceConnection.ts @@ -18,4 +18,8 @@ export const executeCommand = async (id: string, command: string) => { } } return commandHttpService.post(commandPath, payload) +} + +export const registerConnector = async (requestBody: any) => { + return commandHttpService.post("/connector/v1/register", requestBody) } \ No newline at end of file diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 278ac586..76bd9c3a 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -13,6 +13,9 @@ const telemetryObject = { type: "alert", ver: "1.0.0" }; const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const alertPayload = getAlertPayload(req.body); + const userID = (req as any)?.userID; + _.set(alertPayload, "created_by", userID); + _.set(alertPayload, "updated_by", userID); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); @@ -28,8 +31,11 @@ const createAlertHandler = async (req: Request, res: Response, next: NextFunctio const publishAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const { alertId } = req.params; - const rulePayload: Record | null = await getAlertRule(alertId); - if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const ruleModel: Record | null = await getAlertRule(alertId); + if (!ruleModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const rulePayload = ruleModel.toJSON(); + const userID = (req as any)?.userID; + _.set(rulePayload, "updated_by", userID); if (rulePayload.status == "live") { await deleteAlertRule(rulePayload, false); } @@ -87,6 +93,8 @@ const deleteAlertHandler = async (req: Request, res: Response, next: NextFunctio return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); } const rulePayload = ruleModel.toJSON(); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(rulePayload, "updated_by", userID); await deleteAlertRule(rulePayload, hardDelete === "true"); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); @@ -103,12 +111,14 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio const ruleModel = await getAlertRule(alertId); if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } const rulePayload = ruleModel.toJSON(); + const userID = (req as any)?.userID; if (rulePayload.status == "live") { + _.set(rulePayload, "updated_by", userID); await deleteAlertRule(rulePayload, false); await retireAlertSilence(alertId); } const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); - await Alert.update({ ...updatedPayload, status: "draft" }, { where: { id: alertId } }); + await Alert.update({ ...updatedPayload, status: "draft", updated_by: userID }, { where: { id: alertId } }); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); } catch (error: any) { diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index 32e2c531..f424465a 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -20,12 +20,15 @@ const createHandler = async (request: Request, response: Response, next: NextFun const start_date = new Date(startDate); const end_date = new Date(endDate); + const userID = (request as any)?.userID; const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, alert_id: alertId, start_time: start_date, end_time: end_date, + created_by : userID, + updated_by : userID, } const sileneResponse = await Silence.create(silenceBody); updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); @@ -78,10 +81,12 @@ const updateHandler = async (request: Request, response: Response, next: NextFun await updateSilence(silenceObject, payload); const updatedStartTime = new Date(payload.startTime); const updatedEndTime = new Date(payload.endTime); + const userID = (request as any)?.userID; const updatedSilence = { ...silenceObject, start_time: updatedStartTime, - end_time: updatedEndTime + end_time: updatedEndTime, + updated_by: userID, } const silenceResponse = await Silence.update(updatedSilence, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } }) diff --git a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts new file mode 100644 index 00000000..4844ac7e --- /dev/null +++ b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts @@ -0,0 +1,121 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import _ from "lodash"; +import logger from "../../logger"; +import axios from "axios"; +import httpStatus from "http-status"; +import busboy from "busboy"; +import { PassThrough } from "stream"; +import { registerConnector } from "../../connections/commandServiceConnection"; +import { generatePreSignedUrl } from "../GenerateSignedURL/helper"; + +export const apiId = "api.connector.register"; +export const code = "FAILED_TO_REGISTER_CONNECTOR"; + +let resmsgid: string | any; + +const connectorRegisterController = async (req: Request, res: Response) => { + resmsgid = _.get(res, "resmsgid"); + try { + const uploadStreamResponse: any = await uploadStream(req); + const payload = { + relative_path: uploadStreamResponse[0] + } + logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) + const registryResponse = await registerConnector(payload); + logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) + } catch (error: any) { + const errMessage = _.get(error, "response.data.error.message") + logger.error(error, apiId, resmsgid, code); + let errorMessage = error; + const statusCode = _.get(error, "statusCode") + if (!statusCode || statusCode == 500) { + errorMessage = { code, message: errMessage || "Failed to register connector" } + } + ResponseHandler.errorResponse(errorMessage, req, res); + } +}; + +const uploadStream = async (req: Request) => { + return new Promise((resolve, reject) => { + const filePromises: Promise[] = []; + const busboyClient = busboy({ headers: req.headers }); + const relative_path: any[] = []; + let fileCount = 0; + + busboyClient.on("file", async (name: any, file: any, info: any) => { + if (fileCount > 0) { + // If more than one file is detected, reject the request + busboyClient.emit("error", reject({ + code: "FAILED_TO_UPLOAD", + message: "Uploading multiple files are not allowed", + statusCode: 400, + errCode: "BAD_REQUEST" + })); + return + } + fileCount++; + const processFile = async () => { + const fileName = info?.filename; + try { + const preSignedUrl: any = await generatePreSignedUrl("write", [fileName], "connector") + const filePath = preSignedUrl[0]?.filePath + const fileNameExtracted = extractFileNameFromPath(filePath); + relative_path.push(...fileNameExtracted); + const pass = new PassThrough(); + file.pipe(pass); + const fileBuffer = await streamToBuffer(pass); + await axios.put(preSignedUrl[0]?.preSignedUrl, fileBuffer, { + headers: { + "Content-Type": info.mimeType, + "Content-Length": fileBuffer.length, + } + }); + } + catch (err) { + logger.error({ apiId, err, resmsgid, message: "Failed to generate sample urls", code: "FILES_GENERATE_URL_FAILURE" }) + reject({ + code: "FILES_GENERATE_URL_FAILURE", + message: "Failed to generate sample urls", + statusCode: 500, + errCode: "INTERNAL_SERVER_ERROR" + }) + } + }; + filePromises.push(processFile()); + }); + busboyClient.on("close", async () => { + try { + await Promise.all(filePromises); + resolve(relative_path); + } catch (error) { + logger.error({ apiId, error, resmsgid, message: "Fail to upload a file", code: "FAILED_TO_UPLOAD" }) + reject({ + code: "FAILED_TO_UPLOAD", + message: "Fail to upload a file", + statusCode: 400, + errCode: "BAD_REQUEST" + }); + } + }); + busboyClient.on("error", reject); + req.pipe(busboyClient); + }) +} + +const streamToBuffer = (stream: PassThrough): Promise => { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk) => chunks.push(chunk)); + stream.on("end", () => resolve(Buffer.concat(chunks))); + stream.on("error", reject); + }); +}; + +const extractFileNameFromPath = (filePath: string): string[] => { + const regex = /(?<=\/)[^/]+\.[^/]+(?=\/|$)/g; + return filePath.match(regex) || []; +}; + +export default connectorRegisterController; diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index 90808e7d..c7fffa29 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -42,6 +42,9 @@ export const createQueryTemplate = async (req: Request, res: Response) => { } const data = transformRequest(requestBody, templateName); + const userID = (req as any)?.userID; + _.set(data, "created_by", userID); + _.set(data, "updated_by", userID); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) return ResponseHandler.successResponse(req, res, { status: 200, data: { template_id: templateId, template_name: templateName, message: `The query template has been saved successfully` } }); diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 415849fe..46f829e9 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -23,11 +23,10 @@ const errorObject = { } } const apiId = "api.data.in"; -const errorCode = "DATASET_UPDATE_FAILURE" const dataIn = async (req: Request, res: Response) => { - try { - const requestBody = req.body; + + const requestBody = req.body; const datasetId = req.params.datasetId.trim(); const isValidSchema = schemaValidation(requestBody, validationSchema) @@ -35,29 +34,20 @@ const dataIn = async (req: Request, res: Response) => { logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" }) return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_INGESTION_INVALID_INPUT" }, req, res); } - const dataset = await datasetService.getDataset(datasetId, ["id"], true) + const dataset = await datasetService.getDataset(datasetId, ["id", "entry_topic", "api_version", "dataset_config"], true) if (!dataset) { logger.error({ apiId, message: `Dataset with id ${datasetId} not found in live table`, code: "DATASET_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.datasetNotFound, req, res); } - const entryTopic = _.get(dataset, "dataValues.dataset_config.entry_topic") + const { entry_topic, dataset_config, api_version } = dataset + const entryTopic = api_version !== "v2" ? _.get(dataset_config, "entry_topic") : entry_topic if (!entryTopic) { logger.error({ apiId, message: "Entry topic not found", code: "TOPIC_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.topicNotFound, req, res); } - await send(addMetadataToEvents(datasetId, requestBody), _.get(dataset, "dataValues.dataset_config.entry_topic")) + await send(addMetadataToEvents(datasetId, requestBody), entryTopic) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Data ingested successfully" } }); - } - catch (err: any) { - const code = _.get(err, "code") || errorCode - logger.error({ ...err, apiId, code }) - let errorMessage = err; - const statusCode = _.get(err, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: "DATA_INGESTION_FAILED", message: "Failed to ingest data" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } + } const addMetadataToEvents = (datasetId: string, payload: any) => { diff --git a/api-service/src/controllers/DataOut/DataOutController.ts b/api-service/src/controllers/DataOut/DataOutController.ts index a61bf545..dd6f93b6 100644 --- a/api-service/src/controllers/DataOut/DataOutController.ts +++ b/api-service/src/controllers/DataOut/DataOutController.ts @@ -12,44 +12,33 @@ const dataOut = async (req: Request, res: Response) => { const datasetId = req.params?.datasetId; const requestBody = req.body; const msgid = _.get(req, "body.params.msgid"); - try { - const isValidSchema = schemaValidation(requestBody, validationSchema); - if (!isValidSchema?.isValid) { - logger.error({ apiId, datasetId, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) - return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); - } - const isValidQuery: any = await validateQuery(req.body, datasetId); - const query = _.get(req, "body.query", "") - - if (isValidQuery === true && _.isObject(query)) { - const result = await executeNativeQuery(query); - logger.info({ apiId, msgid, requestBody, datasetId, message: "Native query executed successfully" }) - return ResponseHandler.successResponse(req, res, { - status: 200, data: result?.data - }); - } + const isValidSchema = schemaValidation(requestBody, validationSchema); + if (!isValidSchema?.isValid) { + logger.error({ apiId, datasetId, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) + return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); + } + const isValidQuery: any = await validateQuery(req.body, datasetId); + const query = _.get(req, "body.query", "") - if (isValidQuery === true && _.isString(query)) { - const result = await executeSqlQuery({ query }) - logger.info({ apiId, msgid, requestBody, datasetId, message: "SQL query executed successfully" }) - return ResponseHandler.successResponse(req, res, { - status: 200, data: result?.data - }); - } + if (isValidQuery === true && _.isObject(query)) { + const result = await executeNativeQuery(query); + logger.info({ apiId, msgid, requestBody, datasetId, message: "Native query executed successfully" }) + return ResponseHandler.successResponse(req, res, { + status: 200, data: result?.data + }); + } - else { - logger.error({ apiId, msgid, requestBody, datasetId, message: isValidQuery?.message, code: isValidQuery?.code }) - return ResponseHandler.errorResponse({ message: isValidQuery?.message, statusCode: isValidQuery?.statusCode, errCode: isValidQuery?.errCode, code: isValidQuery?.code }, req, res); - } + if (isValidQuery === true && _.isString(query)) { + const result = await executeSqlQuery({ query }) + logger.info({ apiId, msgid, requestBody, datasetId, message: "SQL query executed successfully" }) + return ResponseHandler.successResponse(req, res, { + status: 200, data: result?.data + }); } - catch (err: any) { - logger.error({ ...err, apiId, code: err?.code || "INTERNAL_SERVER_ERROR" }) - let errorMessage = err; - const statusCode = _.get(err, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: "INTERNAL_SERVER_ERROR", message: "Unable to process the query." } - } - ResponseHandler.errorResponse(errorMessage, req, res); + + else { + logger.error({ apiId, msgid, requestBody, datasetId, message: isValidQuery?.message, code: isValidQuery?.code }) + return ResponseHandler.errorResponse({ message: isValidQuery?.message, statusCode: isValidQuery?.statusCode, errCode: isValidQuery?.errCode, code: isValidQuery?.code }, req, res); } } diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index e569bdb3..2a0c64ed 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -4,10 +4,10 @@ import * as _ from "lodash"; import moment from "moment"; import { getDatasourceList } from "../../services/DatasourceService"; import logger from "../../logger"; -import { getDatasourceListFromDruid } from "../../connections/druidConnection"; +import { druidHttpService, getDatasourceListFromDruid } from "../../connections/druidConnection"; import { apiId } from "./DataOutController"; -import { ErrorObject } from "../../types/ResponseModel"; import { Parser } from "node-sql-parser"; +import { obsrvError } from "../../types/ObsrvError"; const parser = new Parser(); const momentFormat = "YYYY-MM-DD HH:MM:SS"; @@ -15,7 +15,7 @@ let dataset_id: string; let requestBody: any; let msgid: string; const errCode = { - notFound: "DATA_OUT_SOURCE_NOT_FOUND", + notFound: "DATASOURCE_NOT_FOUND", invalidDateRange: "DATA_OUT_INVALID_DATE_RANGE" } @@ -56,6 +56,27 @@ const getLimit = (queryLimit: number, maxRowLimit: number) => { return queryLimit > maxRowLimit ? maxRowLimit : queryLimit; }; +const parseSqlQuery = (queryPayload: any) => { + try { + const vocabulary: any = parser.astify(queryPayload?.query); + const isLimitIncludes = JSON.stringify(vocabulary); + if (_.includes(isLimitIncludes, "{{LIMIT}}")) { + return queryPayload?.query + } + const limit = _.get(vocabulary, "limit") + if (limit === null) { + _.set(vocabulary, "limit.value[0].value", queryRules.common.maxResultRowLimit) + _.set(vocabulary, "limit.value[0].type", "number") + let convertToSQL = parser.sqlify(vocabulary); + convertToSQL = convertToSQL.replace(/`/g, "\""); + queryPayload.query = convertToSQL + } + return true + } catch (error) { + logger.warn("Sql parser error", error) + return false + } +} const setQueryLimits = (queryPayload: any) => { if (_.isObject(queryPayload?.query)) { const threshold = _.get(queryPayload, "query.threshold") @@ -78,17 +99,18 @@ const setQueryLimits = (queryPayload: any) => { } if (_.isString(queryPayload?.query)) { - const vocabulary: any = parser.astify(queryPayload?.query); - const isLimitIncludes = JSON.stringify(vocabulary); - if (_.includes(isLimitIncludes, "{{LIMIT}}")) { - return queryPayload?.query - } - const limit = _.get(vocabulary, "limit") - if (limit === null) { - _.set(vocabulary, "limit.value[0].value", queryRules.common.maxResultRowLimit) - _.set(vocabulary, "limit.value[0].type", "number") - const convertToSQL = parser.sqlify(vocabulary); - queryPayload.query = convertToSQL + const validDruidSql = parseSqlQuery(queryPayload) + if (!validDruidSql) { + let query = queryPayload.query; + if (/\blimit\b/i.test(query)) { + return query; + } + const limitRegex = /\bLIMIT\b\s+\d+/i; + if (!limitRegex.test(query)) { + const maxLimit = _.get(queryRules, "common.maxResultRowLimit"); + query += ` LIMIT ${maxLimit}`; + queryPayload.query = query; + } } } } @@ -132,7 +154,7 @@ const validateDateRange = (fromDate: moment.Moment, toDate: moment.Moment, allow } else { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Data range cannnot be more than ${allowedRange} days.`, code: errCode.invalidDateRange }) - throw { message: `Invalid date range! make sure your range cannot be more than ${allowedRange} days`, statusCode: 400, errCode: "BAD_REQUEST", code: errCode.invalidDateRange } as ErrorObject; + throw obsrvError("", errCode.invalidDateRange, `Invalid date range! make sure your range cannot be more than ${allowedRange} days`, "BAD_REQUEST", 400); } }; @@ -155,28 +177,46 @@ const validateQueryRules = (queryPayload: any, limits: any) => { : { message: "Invalid date range! the date range cannot be a null value", statusCode: 400, errCode: "BAD_REQUEST", code: errCode.invalidDateRange }; }; -const getDataSourceRef = async (datasetId: string, granularity?: string) => { +const getDataSourceRef = async (datasetId: string, requestGranularity?: string) => { const dataSources = await getDatasourceList(datasetId) if (_.isEmpty(dataSources)) { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Datasource ${datasetId} not available in datasource live table`, code: errCode.notFound }) - throw { message: `Datasource ${datasetId} not available for querying`, statusCode: 404, errCode: "NOT_FOUND", code: errCode.notFound } as ErrorObject; + throw obsrvError("", errCode.notFound, `Datasource ${datasetId} not available for querying`, "NOT_FOUND", 404); } - const record = dataSources.filter((record: any) => { - const aggregatedRecord = _.get(record, "dataValues.metadata.aggregated") - if (granularity) - return aggregatedRecord && _.get(record, "dataValues.metadata.granularity") === granularity; + const record = dataSources.find((record: any) => { + const metadata = _.get(record, "dataValues.metadata", {}); + const { aggregated, granularity } = metadata; + if (!aggregated) { + return true; + } + return aggregated && requestGranularity ? granularity === requestGranularity : false; }); - return record[0]?.dataValues?.datasource_ref + return _.get(record, ["dataValues", "datasource_ref"]) +} + +const checkSupervisorAvailability = async (datasourceRef: string) => { + const { data } = await druidHttpService.get("/druid/coordinator/v1/loadstatus"); + const datasourceAvailability = _.get(data, datasourceRef) + if (_.isUndefined(datasourceAvailability)) { + throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not available for querying", "NOT_FOUND", 404) + } + if (datasourceAvailability !== 100) { + throw obsrvError("", "DATASOURCE_NOT_FULLY_AVAILABLE", "Datasource not fully available for querying", "RANGE_NOT_SATISFIABLE", 416) + } } const setDatasourceRef = async (datasetId: string, payload: any): Promise => { const granularity = _.get(payload, "context.aggregationLevel") const datasourceRef = await getDataSourceRef(datasetId, granularity); + if (!datasourceRef) { + throw obsrvError("", "DATASOURCE_NOT_FOUND", "Datasource not found to query", "NOT_FOUND", 404) + } + await checkSupervisorAvailability(datasourceRef) const existingDatasources = await getDatasourceListFromDruid(); if (!_.includes(existingDatasources.data, datasourceRef)) { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Dataset ${datasetId} with table ${granularity} is not available for querying`, code: errCode.notFound }) - throw { message: `Dataset ${datasetId} with table ${granularity} is not available for querying`, statusCode: 404, errCode: "NOT_FOUND", code: errCode.notFound } as ErrorObject; + throw obsrvError("", errCode.notFound, `Dataset ${datasetId} with table ${granularity} is not available for querying`, "NOT_FOUND", 404); } if (_.isString(payload?.query)) { payload.query = payload.query.replace(datasetId, datasourceRef) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 9308dd71..9015e9c7 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -40,6 +40,9 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); + const userID = (req as any)?.userID; + _.set(dataset, "created_by", userID); + _.set(dataset, "updated_by", userID); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { if (err?.name === "SequelizeUniqueConstraintError") { diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts index 7b50626c..f89a9829 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -2,7 +2,8 @@ import * as _ from "lodash"; import { DatasetStatus } from "../../types/DatasetModels"; import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; import { config } from "../../configs/Config"; -const version = defaultDatasetConfig.version; +const defaultConfigs = _.cloneDeep(defaultDatasetConfig); +const version = defaultConfigs.version; export const updateRecords = (datasetRecord: Record, newDatasetId: string): void => { const dataset_id = newDatasetId; @@ -10,7 +11,6 @@ export const updateRecords = (datasetRecord: Record, newDatasetId: _.set(datasetRecord, "status", DatasetStatus.Draft) _.set(datasetRecord, "dataset_id", dataset_id) _.set(datasetRecord, "id", dataset_id) - _.set(datasetRecord, "name", dataset_id) _.set(datasetRecord, "version_key", Date.now().toString()) _.set(datasetRecord, "version", version); _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 2867457b..bb96dac2 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -34,6 +34,9 @@ const datasetCreate = async (req: Request, res: Response) => { await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) + const userID = (req as any)?.userID; + _.set(draftDataset, "created_by", userID); + _.set(draftDataset, "updated_by", userID); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json index 13097a8b..15039569 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -36,6 +36,9 @@ "type": "string", "minLength": 1 }, + "client_state": { + "type": "object" + }, "validation_config": { "type": "object", "properties": { @@ -69,6 +72,9 @@ "type": "boolean", "default": true }, + "batch_id": { + "type": "string" + }, "extraction_key": { "type": "string", "default": "events" @@ -161,6 +167,9 @@ "type": "string", "minLength": 1 }, + "required": { + "type": "array" + }, "properties": { "type": "object", "minProperties": 1 @@ -170,8 +179,8 @@ "default": true } }, - "required": ["$schema", "type", "properties", "additionalProperties"], - "additionalProperties": false + "required": ["type", "properties"], + "additionalProperties": true }, "dataset_config": { "type": "object", diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts index 7d06a196..73907183 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -115,7 +115,8 @@ export const migrateExportedDatasetV1 = (requestPayload: Record) => const { drop_duplicates, dedup_key, dedup_period, extraction_key, enabled: is_batch_event } = batch dataset["extraction_config"] = { is_batch_event, extraction_key, dedup_config: { drop_duplicates, dedup_key, dedup_period } } - const { redis_db, redis_db_host, redis_db_port } = defaultDatasetConfig.dataset_config.cache_config; + const defaults = _.cloneDeep(defaultDatasetConfig); + const { redis_db, redis_db_host, redis_db_port } = defaults.dataset_config.cache_config; dataset["dataset_config"] = { indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (type === DatasetType.master) }, keys_config: { data_key, timestamp_key }, diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json index 45820502..da61770c 100644 --- a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json @@ -77,6 +77,9 @@ "type": "boolean", "default": true }, + "batch_id": { + "type": "string" + }, "extraction_key": { "type": "string", "default": "events" diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts new file mode 100644 index 00000000..903d393c --- /dev/null +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -0,0 +1,38 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { executeNativeQuery } from "../../connections/druidConnection"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import vaidationSchema from "./DatasetMetricsValidationSchema.json" +import { schemaValidation } from "../../services/ValidationService"; +import logger from "../../logger"; +import { obsrvError } from "../../types/ObsrvError"; +import axios from "axios"; +import { config } from "../../configs/Config"; + +const getBaseUrl = (url: string) => { + if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") +} + +const datasetMetrics = async (req: Request, res: Response) => { + const isValidSchema = schemaValidation(req.body, vaidationSchema); + if (!isValidSchema?.isValid) { + logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) + throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + } + const { query } = req.body || {}; + const endpoint = query.url; + if (_.startsWith(endpoint, "/prom")) { + query.url = getBaseUrl(endpoint) + const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; + const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) + const data = _.get(apiResponse, "data"); + return res.json(data); + } + else { + const query = _.get(req, ["body", "query", "body", "query"]); + const response = await executeNativeQuery(query); + ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + } +} + +export default datasetMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json b/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json new file mode 100644 index 00000000..ae17e302 --- /dev/null +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "query": { + "type": "object", + "nullable": true + } + }, + "required": [ + "query" + ] +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 2822e87b..d428b02a 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -3,7 +3,7 @@ import httpStatus from "http-status"; import _ from "lodash"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { DatasetDraft } from "../../models/DatasetDraft"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, getV1Connectors } from "../../services/DatasetService"; import { obsrvError } from "../../types/ObsrvError"; import { cipherService } from "../../services/CipherService"; @@ -30,33 +30,31 @@ const datasetRead = async (req: Request, res: Response) => { validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; + const userID = (req as any)?.userID; const attributes = !fields ? defaultFields : _.split(fields, ","); - const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes) + const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userID) : await readDataset(dataset_id, attributes) if (!dataset) { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } if (dataset.connectors_config) { - dataset.connectors_config = dataset.connectors_config.map((connector: any) => ({ - ...connector, - connector_config: JSON.parse(cipherService.decrypt(connector.connector_config)) - })); + dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } -const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { +const readDraftDataset = async (datasetId: string, attributes: string[], userID: string): Promise => { const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); if (draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); - const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset) + const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset, userID) return _.pick(dataset, attributes); } const liveDataset = await datasetService.getDataset(datasetId, undefined, true); if (liveDataset) { - const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) + const dataset = await datasetService.createDraftDatasetFromLive(liveDataset, userID) return _.pick(dataset, attributes); } @@ -65,7 +63,53 @@ const readDraftDataset = async (datasetId: string, attributes: string[]): Promis const readDataset = async (datasetId: string, attributes: string[]): Promise => { const dataset = await datasetService.getDataset(datasetId, attributes, true); - return dataset; + if (!dataset) { + return; + } + const api_version = _.get(dataset, "api_version") + const datasetConfigs: any = {} + const transformations_config = await datasetService.getTransformations(datasetId, ["field_key", "transformation_function", "mode", "metadata"]) + if (api_version !== "v2") { + datasetConfigs["connectors_config"] = await getV1Connectors(datasetId) + datasetConfigs["transformations_config"] = _.map(transformations_config, (config) => { + const section: any = _.get(config, "metadata.section") || _.get(config, "transformation_function.category"); + return { + field_key: _.get(config, "field_key"), + transformation_function: { + ..._.get(config, ["transformation_function"]), + datatype: _.get(config, "metadata._transformedFieldDataType") || _.get(config, "transformation_function.datatype") || "string", + category: datasetService.getTransformationCategory(section), + }, + mode: _.get(config, "mode") + } + }) + } + else { + const v1connectors = await getV1Connectors(datasetId) + const v2connectors = await datasetService.getConnectors(datasetId, ["id", "connector_id", "connector_config", "operations_config"]); + datasetConfigs["connectors_config"] = _.concat(v1connectors, v2connectors) + datasetConfigs["transformations_config"] = transformations_config; + } + return { ...dataset, ...datasetConfigs }; +} + +const processConnectorsConfig = (connectorsConfig: any) => { + return connectorsConfig.map((connector: any) => { + let connector_config = _.get(connector, "connector_config"); + const authMechanism = _.get(connector_config, ["authenticationMechanism"]); + + if (authMechanism && authMechanism.encrypted) { + connector_config = { + ...connector_config, + authenticationMechanism: JSON.parse(cipherService.decrypt(authMechanism.encryptedValues)) + }; + } + + return { + ...connector, + connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector_config)) + }; + }); } export default datasetRead; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 8506afd3..0197d39e 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -18,13 +18,11 @@ const allowedTransitions: Record = { Delete: [DatasetStatus.Draft, DatasetStatus.ReadyToPublish], ReadyToPublish: [DatasetStatus.Draft], Live: [DatasetStatus.ReadyToPublish], - Retire: [DatasetStatus.Live], - Archive: [DatasetStatus.Retired], - Purge: [DatasetStatus.Archived] + Retire: [DatasetStatus.Live] } -const liveDatasetActions = ["Retire", "Archive", "Purge"] +const liveDatasetActions = ["Retire"] -const validateRequest = (req: Request, datasetId: any) => { +const validateRequest = (req: Request, datasetId: any) => { const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) if (!isRequestValid.isValid) { throw obsrvError(datasetId, invalidRequest, isRequestValid.message, "BAD_REQUEST", 400) @@ -32,7 +30,7 @@ const validateRequest = (req: Request, datasetId: any) => { } const validateDataset = (dataset: any, datasetId: any, action: string) => { - + if (_.isEmpty(dataset)) { throw obsrvError(datasetId, datasetNotFound, `Dataset not found for dataset: ${datasetId}`, "NOT_FOUND", 404) } @@ -41,7 +39,7 @@ const validateDataset = (dataset: any, datasetId: any, action: string) => { throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "BAD_REQUEST", 400) } - if(!_.includes(allowedTransitions[action], dataset.status)) { + if (!_.includes(allowedTransitions[action], dataset.status)) { throw obsrvError(datasetId, `DATASET_${_.toUpper(action)}_FAILURE`, `Transition failed for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, "CONFLICT", 409) } @@ -54,28 +52,25 @@ const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); validateRequest(req, dataset_id); - const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version", "name"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const userID = (req as any)?.userID; validateDataset(dataset, dataset_id, status); - switch(status) { + switch (status) { case "Delete": await deleteDataset(dataset); break; case "ReadyToPublish": - await readyForPublish(dataset); + await readyForPublish(dataset, userID); break; case "Live": - await publishDataset(dataset); + await publishDataset(dataset, userID); break; case "Retire": - await retireDataset(dataset); - break; - case "Archive": - await archiveDataset(dataset); - break; - case "Purge": - await purgeDataset(dataset); + await retireDataset(dataset, userID); break; + default: + throw obsrvError(dataset.id, "UNKNOWN_STATUS_TRANSITION", "Unknown status transition requested", "BAD_REQUEST", 400) } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: `Dataset status transition to ${status} successful`, dataset_id } }); @@ -90,19 +85,32 @@ const deleteDataset = async (dataset: Record) => { } -const readyForPublish = async (dataset: Record) => { +const readyForPublish = async (dataset: Record, updated_by: any) => { const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) + defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); + if (_.get(draftDataset, "dataset_config.keys_config")) { + defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config"); + } if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); + defaultConfigs = _.omit(defaultConfigs, "dataset_config.cache_config.redis_db"); } - _.mergeWith(draftDataset,defaultConfigs,draftDataset, (objValue, srcValue) => { + _.set(draftDataset, "updated_by", updated_by); + _.mergeWith(draftDataset, draftDataset, defaultConfigs, (objValue, srcValue, key) => { + if (key === "created_by" || key === "updated_by") { + if (objValue !== "SYSTEM") { + return objValue; + } + return srcValue; + } if (_.isBoolean(objValue) && _.isBoolean(srcValue)) { return objValue; } }); + const datasetValid: Record = schemaValidation(draftDataset, ReadyToPublishSchema) if (!datasetValid.isValid) { throw { @@ -125,10 +133,11 @@ const readyForPublish = async (dataset: Record) => { * * @param dataset */ -const publishDataset = async (dataset: Record) => { +const publishDataset = async (dataset: Record, userID: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record - + _.set(draftDataset, ["created_by"], userID); + _.set(draftDataset, ["updated_by"], userID); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) await datasetService.publishDataset(draftDataset) @@ -138,9 +147,9 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) // 1. Check if there are denorm fields and dependent master datasets are published const denormConfig = _.get(draftDataset, "denorm_config") - if(denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { + if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { const datasetIds = _.map(denormConfig.denorm_fields, "dataset_id") - if(_.includes(datasetIds, draftDataset.id)) { + if (_.includes(datasetIds, draftDataset.id)) { throw { code: "SELF_REFERENCING_MASTER_DATA", message: `The denorm master dataset is self-referencing itself`, @@ -148,26 +157,26 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) statusCode: 409 } } - const masterDatasets = await datasetService.findDatasets({id: datasetIds, type: "master"}, ["id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ id: datasetIds, type: "master" }, ["id", "status", "dataset_config", "api_version"]) const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) - const datasetStatus : Record = { + const datasetStatus: Record = { dataset_id: denormField.dataset_id, exists: (md) ? true : false, - isLive: (md) ? md.status === "Live" : false, + isLive: (md) ? md.status === "Live" : false, status: (md) ? md.status : false } - if(!_.isEmpty(md)){ - if(md.api_version === "v2") - datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); - else - datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); + if (!_.isEmpty(md)) { + if (md.api_version === "v2") + datasetStatus["denorm_field"] = _.merge(denormField, { redis_db: md.dataset_config.cache_config.redis_db }); + else + datasetStatus["denorm_field"] = _.merge(denormField, { redis_db: md.dataset_config.redis_db }); } return datasetStatus; }) - const invalidMasters = _.filter(masterDatasetsStatus, {isLive: false}) - if(_.size(invalidMasters) > 0) { + const invalidMasters = _.filter(masterDatasetsStatus, { isLive: false }) + if (_.size(invalidMasters) > 0) { const invalidIds = _.map(invalidMasters, "dataset_id") throw { code: "DEPENDENT_MASTER_DATA_NOT_LIVE", @@ -178,9 +187,10 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) } // 2. Populate redis db for denorm + const defaults = _.cloneDeep(defaultDatasetConfig) draftDataset["denorm_config"] = { - redis_db_host: defaultDatasetConfig.denorm_config.redis_db_host, - redis_db_port: defaultDatasetConfig.denorm_config.redis_db_port, + redis_db_host: defaults.denorm_config.redis_db_host, + redis_db_port: defaults.denorm_config.redis_db_port, denorm_fields: _.map(masterDatasetsStatus, "denorm_field") } } @@ -188,12 +198,20 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) const updateMasterDataConfig = async (draftDataset: Record) => { if (draftDataset.type === "master") { - const dataset_config = _.get(draftDataset, "dataset_config") - const datasetCacheConfig = _.get(defaultDatasetConfig, "dataset_config.cache_config") - draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } + const datasetConfig = _.get(draftDataset, "dataset_config") + const defaultConfigs = _.cloneDeep(defaultDatasetConfig); + + const defaultCacheConfig = _.get(defaultConfigs, "dataset_config.cache_config"); + const currentCacheConfig = _.get(datasetConfig, "cache_config"); + draftDataset.dataset_config = { + ...datasetConfig, cache_config: { + ...defaultCacheConfig, ..._.omit(currentCacheConfig, "redis_db"), + redis_db: _.get(currentCacheConfig, "redis_db") + } + }; if (draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() - if(_.isEmpty(results)) { + if (_.isEmpty(results)) { throw { code: "REDIS_DB_INDEX_FETCH_FAILED", message: `Unable to fetch the redis db index for the master data`, @@ -207,10 +225,10 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } -const retireDataset = async (dataset: Record) => { +const retireDataset = async (dataset: Record, updated_by: any) => { await canRetireIfMasterDataset(dataset); - await datasetService.retireDataset(dataset); + await datasetService.retireDataset(dataset, updated_by); await restartPipeline(dataset); } @@ -222,14 +240,14 @@ const canRetireIfMasterDataset = async (dataset: Record) => { const liveDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live }, ["dataset_config", "dataset_id"]) || [] const draftDatasets = await datasetService.findDraftDatasets({ status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, ["denorm_config", "id", "status"]) || [] const allDatasets = _.union(liveDatasets, draftDatasets) - const extractDenormFields = _.map(allDatasets, function(depDataset) { - return {dataset_id: _.get(depDataset, "id"), status: _.get(depDataset, "status"), denorm_datasets: _.map(_.get(depDataset, "denorm_config.denorm_fields"), "dataset_id")} + const extractDenormFields = _.map(allDatasets, function (depDataset) { + return { dataset_id: _.get(depDataset, "id"), status: _.get(depDataset, "status"), denorm_datasets: _.map(_.get(depDataset, "denorm_config.denorm_fields"), "dataset_id") } }) - const deps = _.filter(extractDenormFields, function(depDS) { return _.includes(depDS.denorm_datasets, dataset.id)}) + const deps = _.filter(extractDenormFields, function (depDS) { return _.includes(depDS.denorm_datasets, dataset.id) }) if (_.size(deps) > 0) { const denormErrMsg = `Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset` - throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function(o) { return _.omit(o, "denorm_datasets")})) + throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function (o) { return _.omit(o, "denorm_datasets") })) } } } @@ -238,14 +256,4 @@ export const restartPipeline = async (dataset: Record) => { return executeCommand(dataset.id, "RESTART_PIPELINE") } -const archiveDataset = async (dataset: Record) => { - - throw obsrvError(dataset.id, "ARCHIVE_NOT_IMPLEMENTED", "Archive functionality is not implemented", "NOT_IMPLEMENTED", 501) -} - -const purgeDataset = async (dataset: Record) => { - - throw obsrvError(dataset.id, "PURGE_NOT_IMPLEMENTED", "Purge functionality is not implemented", "NOT_IMPLEMENTED", 501) -} - export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index 281c62ac..d76acbb8 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -14,11 +14,17 @@ }, "status": { "type": "string", - "enum": ["Draft"] + "enum": [ + "Draft" + ] }, "type": { "type": "string", - "enum": ["event", "transaction", "master"] + "enum": [ + "event", + "transaction", + "master" + ] }, "name": { "type": "string", @@ -36,10 +42,16 @@ }, "mode": { "type": "string", - "enum": ["Strict", "IgnoreNewFields"] + "enum": [ + "Strict", + "IgnoreNewFields" + ] } }, - "required": ["validate", "mode"] + "required": [ + "validate", + "mode" + ] }, "extraction_config": { "type": "object", @@ -78,9 +90,15 @@ "dedup_key": { "minLength": 1 } - } + }, + "required": [ + "dedup_key", + "dedup_period" + ] }, - "required": ["drop_duplicates", "dedup_key", "dedup_period"], + "required": [ + "drop_duplicates" + ], "additionalProperties": false } }, @@ -133,9 +151,15 @@ "dedup_key": { "minLength": 1 } - } + }, + "required": [ + "dedup_key", + "dedup_period" + ] }, - "required": ["drop_duplicates", "dedup_key", "dedup_period"], + "required": [ + "drop_duplicates" + ], "additionalProperties": false }, "data_schema": { @@ -160,8 +184,11 @@ "type": "boolean" } }, - "required": ["type", "properties"], - "additionalProperties": false + "required": [ + "type", + "properties" + ], + "additionalProperties": true }, "denorm_config": { "type": "object", @@ -200,17 +227,27 @@ }, "oneOf": [ { - "required": ["dataset_id", "denorm_out_field", "denorm_key"] + "required": [ + "dataset_id", + "denorm_out_field", + "denorm_key" + ] }, { - "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] + "required": [ + "dataset_id", + "denorm_out_field", + "jsonata_expr" + ] } ], "additionalProperties": false } } }, - "required": ["denorm_fields"], + "required": [ + "denorm_fields" + ], "additionalProperties": false }, "router_config": { @@ -221,7 +258,9 @@ "minLength": 1 } }, - "required": ["topic"], + "required": [ + "topic" + ], "additionalProperties": false }, "dataset_config": { @@ -253,9 +292,13 @@ "default": false } }, - "required": ["olap_store_enabled", "lakehouse_enabled", "cache_enabled"], + "required": [ + "olap_store_enabled", + "lakehouse_enabled", + "cache_enabled" + ], "additionalProperties": false - }, + }, "keys_config": { "type": "object", "properties": { @@ -273,7 +316,10 @@ "minLength": 1 } }, - "required": ["data_key", "timestamp_key"], + "required": [ + "data_key", + "timestamp_key" + ], "additionalProperties": false }, "cache_config": { @@ -290,11 +336,18 @@ "type": "integer" } }, - "required": ["redis_db_host", "redis_db_port"], + "required": [ + "redis_db_host", + "redis_db_port" + ], "additionalProperties": false } }, - "required": ["indexing_config", "keys_config", "cache_config"], + "required": [ + "indexing_config", + "keys_config", + "cache_config" + ], "additionalProperties": false }, "tags": { @@ -324,8 +377,7 @@ "minLength": 1 }, "expr": { - "type": "string", - "minLength": 1 + "type": "string" }, "condition": { "type": "object", @@ -335,11 +387,13 @@ "minLength": 1 }, "expr": { - "type": "string", - "minLength": 1 + "type": "string" } }, - "required": ["type", "expr"], + "required": [ + "type", + "expr" + ], "additionalProperties": false }, "datatype": { @@ -347,19 +401,34 @@ }, "category": { "type": "string", - "enum": ["pii", "transform", "derived"] + "enum": [ + "pii", + "transform", + "derived" + ] } }, - "required": ["type", "expr", "category"], + "required": [ + "type", + "expr", + "category" + ], "additionalProperties": false }, "mode": { "type": "string", - "enum": ["Strict", "Lenient"] + "enum": [ + "Strict", + "Lenient" + ] } }, "additionalProperties": false, - "required": ["field_key", "transformation_function", "mode"] + "required": [ + "field_key", + "transformation_function", + "mode" + ] } }, "connectors_config": { @@ -376,7 +445,14 @@ "minLength": 1 }, "connector_config": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "operations_config": { "type": "object" @@ -387,7 +463,12 @@ } }, "additionalProperties": false, - "required": ["id", "connector_id", "connector_config", "version"] + "required": [ + "id", + "connector_id", + "connector_config", + "version" + ] } } }, @@ -408,7 +489,9 @@ "minLength": 1 } }, - "required": ["data_key"] + "required": [ + "data_key" + ] } } } @@ -432,4 +515,4 @@ "transformations_config", "connectors_config" ] -} +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 274ad3c4..e9fea051 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -39,11 +39,11 @@ const validateDataset = (dataset: Record | null, req: Request) => { if (dataset) { if (dataset.api_version !== "v2") { throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "NOT_FOUND", 404) - } - if(dataset.version_key !== versionKey) { + } + if (dataset.version_key !== versionKey) { throw obsrvError(datasetId, "DATASET_OUTDATED", "The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates", "CONFLICT", 409) } - if(!_.includes([DatasetStatus.Draft, DatasetStatus.ReadyToPublish], dataset.status)) { + if (!_.includes([DatasetStatus.Draft, DatasetStatus.ReadyToPublish], dataset.status)) { throw obsrvError(datasetId, "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE", "Dataset cannot be updated as it is not in draft state", "BAD_REQUEST", 400) } } else { @@ -52,74 +52,154 @@ const validateDataset = (dataset: Record | null, req: Request) => { } const datasetUpdate = async (req: Request, res: Response) => { - + await validateRequest(req) const datasetReq = req.body.request; const datasetModel = await datasetService.getDraftDataset(datasetReq.dataset_id) validateDataset(datasetModel, req) - + const draftDataset = mergeDraftDataset(datasetModel, datasetReq); + const userID = (req as any)?.userID; + _.set(draftDataset, "updated_by", userID) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { + const prev_dataset_config = _.get(datasetModel, ["dataset_config"]) + const currentSchema = _.get(datasetModel, 'data_schema') + const fieldsRemoved = (datasetReq.data_schema) ? getMissingFieldsInNewSchema(datasetReq.data_schema, currentSchema) : [] const dataset: Record = { version_key: Date.now().toString(), name: datasetReq.name || _.get(datasetModel, ["name"]), id: _.get(datasetModel, ["id"]) } - if(datasetReq.validation_config) dataset["validation_config"] = datasetReq.validation_config - if(datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config - if(datasetReq.dedup_config) dataset["dedup_config"] = datasetReq.dedup_config - if(datasetReq.data_schema) dataset["data_schema"] = datasetReq.data_schema - if(datasetReq.dataset_config) dataset["dataset_config"] = datasetReq.dataset_config - if(datasetReq.transformations_config) - dataset["transformations_config"] = mergeTransformationsConfig(_.get(datasetModel, ["transformations_config"]), datasetReq.transformations_config) - if(datasetReq.denorm_config) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config) - if(datasetReq.connectors_config) dataset["connectors_config"] = mergeConnectorsConfig(_.get(datasetModel, ["connectors_config"]), datasetReq.connectors_config) - if(datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) - if(datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data - if(datasetReq.type) dataset["type"] = datasetReq.type - + if (datasetReq.validation_config) dataset["validation_config"] = datasetReq.validation_config + if (datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config + if (datasetReq.dedup_config) dataset["dedup_config"] = datasetReq.dedup_config + if (datasetReq.data_schema) dataset["data_schema"] = datasetReq.data_schema + if (datasetReq.dataset_config) dataset["dataset_config"] = { ...prev_dataset_config, ...datasetReq.dataset_config } + if (datasetReq.transformations_config || fieldsRemoved.length > 0) + dataset["transformations_config"] = mergeTransformationsConfig(_.get(datasetModel, ["transformations_config"]), datasetReq.transformations_config, fieldsRemoved) + if (datasetReq.denorm_config || fieldsRemoved.length > 0) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config, fieldsRemoved) + if (datasetReq.connectors_config) dataset["connectors_config"] = mergeConnectorsConfig(_.get(datasetModel, ["connectors_config"]), datasetReq.connectors_config) + if (datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) + if (datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data + if (datasetReq.type) dataset["type"] = datasetReq.type + if (fieldsRemoved.length > 0) { + const keys_config = _.get(dataset["dataset_config"] ? dataset["dataset_config"] : prev_dataset_config, ["keys_config"]) + let modified = false; + if (_.includes(fieldsRemoved, keys_config['data_key'])) { + modified = true; + keys_config['data_key'] = ''; + } + if (_.includes(fieldsRemoved, keys_config['partition_key'])) { + modified = true; + keys_config['partition_key'] = ''; + } + if (_.includes(fieldsRemoved, keys_config['timestamp_key'])) { + modified = true; + keys_config['timestamp_key'] = ''; + } + if (modified) { + if (dataset["dataset_config"]) { + _.set(dataset["dataset_config"], 'keys_config', keys_config) + } else { + const keys_config = _.get(prev_dataset_config, ["keys_config"]) + dataset["dataset_config"] = { ...prev_dataset_config } + _.set(dataset["dataset_config"], 'keys_config', keys_config) + } + } + } return dataset; } -const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any) => { - const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.field_key") - const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") +const getMissingFieldsInNewSchema = (newSchema: any, oldSchema: any) => { + + const getRemovedPropertiesFieldsNested = (oldProperties: Record, newProperties: Record, path: string[] = []): string[] => { + let removedFields: string[] = []; + for (const key in oldProperties) { + const fullPath = [...path, key].join('.'); + if (!(key in newProperties)) { + removedFields.push(fullPath); + if (typeof oldProperties[key] === 'object' && oldProperties[key].properties) { + removedFields = removedFields.concat(getRemovedPropertiesFieldsNested(oldProperties[key].properties || {}, {}, [...path, key])); + } + } + else if (typeof oldProperties[key] === 'object' && typeof newProperties[key] === 'object') { + removedFields = removedFields.concat( + getRemovedPropertiesFieldsNested( + oldProperties[key].properties || {}, + newProperties[key].properties || {}, + [...path, key] + ) + ); + } + } + return removedFields; + }; - return _.unionWith( - addConfigs, - _.reject(currentConfigs, (config) => { return _.includes(removeConfigs, config.field_key)}), - (a, b) => { - return a.field_key === b.field_key - } - ) + + const getRemovedPropertiesFields = (oldSchema: any, newSchema: any): string[] => { + const oldProperties = oldSchema.properties || {}; + const newProperties = newSchema.properties || {}; + return getRemovedPropertiesFieldsNested(oldProperties, newProperties); + } + + // Example usage + const removedFieldsNested = getRemovedPropertiesFields(oldSchema, newSchema); + return removedFieldsNested } -const mergeDenormConfig = (currentConfig: any, newConfig: any) => { +const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any, fieldsRemoved: string[]) => { - const removeConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "remove"}), "value.denorm_out_field") - const addConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "upsert"}), "value") + let updatedConfigs = currentConfigs; + if (newConfigs && newConfigs.length) { + const removeConfigs = _.map(_.filter(newConfigs, { action: "remove" }), "value.field_key") + const addConfigs = _.map(_.filter(newConfigs, { action: "upsert" }), "value") - const denormFields = _.unionWith( - addConfigs, - _.reject(currentConfig.denorm_fields, (config) => { return _.includes(removeConfigs, config.denorm_out_field)}), - (a, b) => { - return a.denorm_out_field === b.denorm_out_field - } - ) + updatedConfigs = _.unionWith( + addConfigs, + _.reject(currentConfigs, (config) => { return _.includes(removeConfigs, config.field_key) }), + (a, b) => { + return a.field_key === b.field_key + } + ) + } + if (fieldsRemoved.length > 0) { + updatedConfigs = _.reject(updatedConfigs, (config) => { return _.includes(fieldsRemoved, config.field_key) }) + } + return updatedConfigs +} + +const mergeDenormConfig = (currentConfig: any, newConfig: any, fieldsRemoved: string[]) => { + + let updatedConfigs = currentConfig.denorm_fields; + if (_.get(newConfig, "denorm_fields")) { + const removeConfigs = _.map(_.filter(newConfig.denorm_fields, { action: "remove" }), "value.denorm_out_field") + const addConfigs = _.map(_.filter(newConfig.denorm_fields, { action: "upsert" }), "value") + + updatedConfigs = _.unionWith( + addConfigs, + _.reject(currentConfig.denorm_fields, (config) => { return _.includes(removeConfigs, config.denorm_out_field) }), + (a, b) => { + return a.denorm_out_field === b.denorm_out_field + } + ) + } + if (fieldsRemoved.length > 0) { + updatedConfigs = _.reject(updatedConfigs, (config) => { return _.includes(fieldsRemoved, config.denorm_key) }) + } return { - denorm_fields: denormFields + denorm_fields: updatedConfigs } } const mergeConnectorsConfig = (currConfigs: any, newConfigs: any) => { - const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.connector_id") - const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + const removeConfigs = _.map(_.filter(newConfigs, { action: "remove" }), "value.connector_id") + const addConfigs = _.map(_.filter(newConfigs, { action: "upsert" }), "value") return _.unionWith( _.map(addConfigs, (config) => { @@ -131,17 +211,17 @@ const mergeConnectorsConfig = (currConfigs: any, newConfigs: any) => { version: config.version } }), - _.reject(currConfigs, (config) => { return _.includes(removeConfigs, config.connector_id)}), - (a, b) => { + _.reject(currConfigs, (config) => { return _.includes(removeConfigs, config.connector_id) }), + (a, b) => { return a.connector_id === b.connector_id - } + } ) } const mergeTags = (currentTags: any, newConfigs: any) => { - const tagsToRemove = _.map(_.filter(newConfigs, {action: "remove"}), "value") - const tagsToAdd = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + const tagsToRemove = _.map(_.filter(newConfigs, { action: "remove" }), "value") + const tagsToAdd = _.map(_.filter(newConfigs, { action: "upsert" }), "value") return _.union(_.pullAll(currentTags, tagsToRemove), tagsToAdd) } diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index b31fe674..e6ca971e 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -170,8 +170,8 @@ "default": true } }, - "required": ["$schema", "type", "properties", "additionalProperties"], - "additionalProperties": false + "required": ["type", "properties"], + "additionalProperties": true }, "dataset_config": { "type": "object", @@ -224,7 +224,7 @@ "additionalProperties": false } }, - "required": ["indexing_config", "keys_config"], + "required": [], "additionalProperties": false }, "transformations_config": { diff --git a/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts index a37e5d13..ce3c7bde 100644 --- a/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts +++ b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -32,7 +32,7 @@ const dataSchema = async (req: Request, res: Response) => { const request = req.body.request const dataSchemaSpec = schemaGenerate(request.data, request.config) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataSchemaSpec }); - + } const schemaGenerate = (sample: Map[], config: Record): any => { @@ -46,13 +46,15 @@ const schemaGenerate = (sample: Map[], config: Record) result.schema = removeFormats(result.schema) return result } else { - let schema = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); + // eslint-disable-next-line + let { schema, removedKeys } = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); schema = schemaArrayValidator.validate(schema) const schemaCardinalityAnalyser = new SchemaCardinalityAnalyser(sample, schema) rollupInfo = schemaCardinalityAnalyser.analyse() const result = process(schema, dataset) result.schema = removeNonIndexColumns(result.schema) result.schema = removeFormats(result.schema) + !_.isEmpty(removedKeys) && _.set(result, "removedKeys", removedKeys) return result } } diff --git a/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts index 6182f9b3..6454b1fa 100644 --- a/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts +++ b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts @@ -6,20 +6,20 @@ import logger from "../../logger"; import { ErrorObject } from "../../types/ResponseModel"; import { schemaValidation } from "../../services/ValidationService"; import GenerateURL from "./GenerateSignedURLValidationSchema.json" -import { cloudProvider } from "../../services/CloudServices"; import { config } from "../../configs/Config"; import { URLAccess } from "../../types/SampleURLModel"; -import { v4 as uuidv4 } from "uuid"; -import path from "path"; +import { generatePreSignedUrl } from "./helper"; export const apiId = "api.files.generate-url" export const code = "FILES_GENERATE_URL_FAILURE" const maxFiles = config.presigned_url_configs.maxFiles +let containerType: string; const generateSignedURL = async (req: Request, res: Response) => { const requestBody = req.body const msgid = _.get(req, ["body", "params", "msgid"]); const resmsgid = _.get(res, "resmsgid"); + containerType = _.get(req, ["body", "request", "type"]); try { const isRequestValid: Record = schemaValidation(req.body, GenerateURL) if (!isRequestValid.isValid) { @@ -46,21 +46,7 @@ const generateSignedURL = async (req: Request, res: Response) => { errCode: "BAD_REQUEST" } as ErrorObject, req, res); } - - const { filesList, updatedFileNames } = transformFileNames(files, access) - logger.info(`Updated file names with path:${updatedFileNames}`) - - const urlExpiry: number = getURLExpiry(access) - const preSignedUrls = await Promise.all(cloudProvider.generateSignedURLs(config.cloud_config.container, updatedFileNames, access, urlExpiry)) - const signedUrlList = _.map(preSignedUrls, list => { - const fileNameWithUid = _.keys(list)[0] - return { - filePath: getFilePath(fileNameWithUid), - fileName: filesList.get(fileNameWithUid), - preSignedUrl: _.values(list)[0] - } - }) - + const signedUrlList = await generatePreSignedUrl(access, files, containerType) logger.info({ apiId, requestBody, msgid, resmsgid, response: signedUrlList, message: `Sample urls generated successfully for files:${files}` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: signedUrlList }) } catch (error: any) { @@ -74,44 +60,6 @@ const generateSignedURL = async (req: Request, res: Response) => { } } -const getFilePath = (file: string) => { - return `${config.cloud_config.container}/${config.presigned_url_configs.service}/user_uploads/${file}` -} - -const transformFileNames = (fileList: Array, access: string): Record => { - if (access === URLAccess.Read) { - return transformReadFiles(fileList) - } - return transformWriteFiles(fileList) -} - -const transformReadFiles = (fileNames: Array) => { - const fileMap = new Map(); - const updatedFileNames = _.map(fileNames, file => { - fileMap.set(file, file) - return getFilePath(file) - }) - return { filesList: fileMap, updatedFileNames } -} - -const transformWriteFiles = (fileNames: Array) => { - const fileMap = new Map(); - const updatedFileNames = _.map(fileNames, file => { - const uuid = uuidv4().replace(/-/g, "").slice(0, 6); - const ext = path.extname(file) - const baseName = path.basename(file, ext) - const updatedFileName = `${baseName}_${uuid}${ext}` - fileMap.set(updatedFileName, file) - return getFilePath(updatedFileName) - }) - return { filesList: fileMap, updatedFileNames } - -} - -const getURLExpiry = (access: string) => { - return access === URLAccess.Read ? config.presigned_url_configs.read_storage_url_expiry : config.presigned_url_configs.write_storage_url_expiry -} - const checkLimitExceed = (files: Array): boolean => { return _.size(files) > maxFiles } diff --git a/api-service/src/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json index 553a784e..77db2e33 100644 --- a/api-service/src/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json +++ b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json @@ -42,6 +42,13 @@ "read", "write" ] + }, + "type": { + "type": "string", + "enum": [ + "dataset", + "connector" + ] } }, "additionalProperties": false, diff --git a/api-service/src/controllers/GenerateSignedURL/helper.ts b/api-service/src/controllers/GenerateSignedURL/helper.ts new file mode 100644 index 00000000..b6601f2a --- /dev/null +++ b/api-service/src/controllers/GenerateSignedURL/helper.ts @@ -0,0 +1,64 @@ +import { config } from "../../configs/Config"; +import * as _ from "lodash"; +import { cloudProvider } from "../../services/CloudServices"; +import { URLAccess } from "../../types/SampleURLModel"; +import path from "path"; +import { v4 as uuidv4 } from "uuid"; + +export const generatePreSignedUrl = async (access: string, files: any, containerType: string) => { + const { filesList, updatedFileNames } = transformFileNames(files, access, containerType); + const urlExpiry: number = getURLExpiry(access); + const preSignedUrls = await Promise.all(cloudProvider.generateSignedURLs(config.cloud_config.container, updatedFileNames, access, urlExpiry)); + const signedUrlList = _.map(preSignedUrls, list => { + const fileNameWithUid = _.keys(list)[0]; + return { + filePath: getFilePath(fileNameWithUid, containerType), + fileName: filesList.get(fileNameWithUid), + preSignedUrl: _.values(list)[0] + }; + }); + return signedUrlList; +} + +const getFilePath = (file: string, containerType: string) => { + const datasetUploadPath = `${config.presigned_url_configs.service}/user_uploads/${file}`; + const connectorUploadPath = `${config.cloud_config.container_prefix}/${file}`; + const paths: Record = { + "dataset": datasetUploadPath, + "connector": connectorUploadPath + }; + return paths[containerType] || datasetUploadPath; +} + +const transformFileNames = (fileList: Array, access: string, containerType: string): Record => { + if (access === URLAccess.Read) { + return transformReadFiles(fileList, containerType); + } + return transformWriteFiles(fileList, containerType); +}; + +const transformReadFiles = (fileNames: Array, containerType: string) => { + const fileMap = new Map(); + const updatedFileNames = fileNames.map(file => { + fileMap.set(file, file); + return getFilePath(file, containerType); + }); + return { filesList: fileMap, updatedFileNames }; +}; + +const transformWriteFiles = (fileNames: Array, containerType: string) => { + const fileMap = new Map(); + const updatedFileNames = fileNames.map(file => { + const uuid = uuidv4().replace(/-/g, "").slice(0, 6); + const ext = path.extname(file); + const baseName = path.basename(file, ext); + const updatedFileName = `${baseName}_${uuid}${ext}`; + fileMap.set(updatedFileName, file); + return getFilePath(updatedFileName, containerType); + }); + return { filesList: fileMap, updatedFileNames }; +}; + +const getURLExpiry = (access: string) => { + return access === URLAccess.Read ? config.presigned_url_configs.read_storage_url_expiry : config.presigned_url_configs.write_storage_url_expiry; +} \ No newline at end of file diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index f6db4fd4..d8850436 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,6 +12,9 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; + const userID = (request as any)?.userID; + _.set(payload, "created_by", userID); + _.set(payload, "updated_by", userID); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) @@ -32,6 +35,8 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } + const userID = (request as any)?.userID; + _.set(updatedPayload, "updated_by", userID); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -74,7 +79,8 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - await Notification.update({ status: "retired" }, { where: { id } }) + const userID = (request as any)?.userID; + await Notification.update({ status: "retired", updated_by: userID }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) @@ -91,7 +97,8 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - Notification.update({ status: "live" }, { where: { id } }); + const userID = (request as any)?.userID; + Notification.update({ status: "live", updated_by: userID }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) diff --git a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts index 114d370e..350b25f7 100644 --- a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts +++ b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts @@ -38,7 +38,8 @@ export const updateQueryTemplate = async (req: Request, res: Response) => { logger.error({ apiId, msgid, resmsgid, templateId, requestBody: req?.body, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, code: "QUERY_TEMPLATE_INVALID_INPUT" }) return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, errCode: "BAD_REQUEST", code: "QUERY_TEMPLATE_INVALID_INPUT" }, req, res) } - + const userID = (req as any)?.userID; + requestBody.request.updated_by = userID; await QueryTemplate.update(requestBody?.request, { where: { template_id: templateId } }) logger.info({ apiId, msgid, resmsgid, templateId, requestBody, message: `Query template updated successfully` }) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Query template updated successfully", templateId } }); diff --git a/api-service/src/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts index 99aee816..67332b37 100644 --- a/api-service/src/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -5,6 +5,7 @@ import { onFailure, onObsrvFailure, onSuccess } from "../metrics/prometheus/help import moment from "moment"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; +import logger from "../logger"; const ResponseHandler = { successResponse: (req: Request, res: Response, result: Result) => { @@ -15,8 +16,8 @@ const ResponseHandler = { entity && onSuccess(req, res) }, - routeNotFound: (req: Request, res: Response, next: NextFunction) => { - next({ statusCode: httpStatus.NOT_FOUND, message: httpStatus["404"], errCode: httpStatus["404_NAME"] }); + routeNotFound: (req: Request, res: Response) => { + ResponseHandler.obsrvErrorResponse({ statusCode: httpStatus.NOT_FOUND, message: "Route not found", errCode: httpStatus["404_NAME"], code: "ROUTE_NOT_FOUND", err: undefined, data: "", datasetId: "" }, req, res) }, refactorResponse: ({ id = "api", ver = "v2", params = { status: "SUCCESS" }, responseCode = httpStatus["200_NAME"], result = {}, msgid = "", resmsgid = "" }): IResponse => { @@ -26,6 +27,8 @@ const ResponseHandler = { errorResponse: (error: Record, req: Request, res: Response) => { const { statusCode, message, errCode, code = "INTERNAL_SERVER_ERROR", trace = "" } = error; + const sanitizedError = { ...error, proxyAuthKey: "REDACTED" }; + logger.error(sanitizedError) const { id, entity, body } = req as any; const msgid = _.get(body, ["params", "msgid"]) const resmsgid = _.get(res, "resmsgid") @@ -37,12 +40,14 @@ const ResponseHandler = { obsrvErrorResponse: (error: ObsrvError, req: Request, res: Response) => { const { statusCode, message, errCode, code = "INTERNAL_SERVER_ERROR", data } = error; + const sanitizedError = { ...error, proxyAuthKey: "REDACTED" }; + logger.error(sanitizedError) const { id, entity, body } = req as any; const msgid = _.get(body, ["params", "msgid"]) const resmsgid = _.get(res, "resmsgid") const response = ResponseHandler.refactorResponse({ id, msgid, params: { status: "FAILED" }, responseCode: errCode || httpStatus["500_NAME"], resmsgid, result: data }) res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json({ ...response, error: { code, message } }); - entity && onObsrvFailure(req,res,error) + entity && onObsrvFailure(req, res, error) }, setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts new file mode 100644 index 00000000..10fda3e2 --- /dev/null +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -0,0 +1,144 @@ +import { Request, Response, NextFunction } from "express"; +import jwt from "jsonwebtoken"; +import { ResponseHandler } from "../helpers/ResponseHandler"; +import { config } from "../configs/Config"; +import _ from "lodash"; +import userPermissions from "./userPermissions.json"; +import httpStatus from "http-status"; +import { userService } from "../services/UserService"; +interface AccessControl { + apiGroups: { + [key: string]: string[]; + }, + roles: { + [key: string]: string[]; + } +} + +const accessControl: AccessControl = userPermissions; + +const errorHandler = (statusCode: number, message: string, req: Request, res: Response) => { + const errorMapping: Record = { + 401: { + errCode: httpStatus["401_NAME"], + code: "UNAUTHORIZED ACCESS", + }, + 403: { + errCode: httpStatus["403_NAME"], + code: "FORBIDDEN ACCESS", + }, + }; + + const { errCode, code } = errorMapping[statusCode]; + + return ResponseHandler.errorResponse( + { + statusCode, + errCode, + message, + code, + }, + req, + res + ); +}; + + +const checkAccess = (decoded: any, action: string, req: Request, res: Response) => { + if (decoded.roles) { + const hasAccess = decoded.roles.some((role: string) => { + const apiGroups = accessControl.roles[role]; + return apiGroups?.some((apiGroup: string) => + accessControl.apiGroups[apiGroup]?.includes(action) + ); + }); + + if (hasAccess) { + return true; + } else { + const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { + const apiGroups = accessControl.roles[role]; + return apiGroups?.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); + }); + + const rolesMessage = rolesWithAccess.length > 0 + ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` + : "No roles have this action"; + + const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; + errorHandler(403, errorMessage, req, res); + return false; + } + } + + errorHandler(403, "Access denied. User does not have permission to perform this action.", req, res); + return false; +}; + +const basicToken = (token: string, req: Request, res: Response, next: NextFunction) => { + try { + const decoded = jwt.verify(token, config.user_token_public_key); + + if (!decoded || !_.isObject(decoded)) { + return errorHandler(401, "Token verification failed or invalid token", req, res); + } + + (req as any).userID = decoded.id; + const action = (req as any).id; + + if (checkAccess(decoded, action, req, res)) { + return next(); + } + } catch (error) { + return errorHandler(401, "Token verification error", req, res); + } +}; + +const keycloakTokenVerify = async (token: string, req: Request, res: Response, next: NextFunction) => { + try { + const decoded = jwt.decode(token); + if (decoded && _.isObject(decoded)) { + (req as any).userID = decoded.sub; + const action = (req as any).id; + const userCondition = { id: decoded.sub }; + const userDetails = ["roles", "user_name"]; + const user = await userService.getUser(userCondition, userDetails); + + if (!user) { + return errorHandler(404, "User not found", req, res); + } + + if (checkAccess(user, action, req, res)) { + return next(); + } + } + } catch (error) { + return errorHandler(401, "Token decode error", req, res); + } +}; + +export default { + name: "rbac:middleware", + handler: () => async (req: Request, res: Response, next: NextFunction) => { + try { + if (_.lowerCase(config.is_RBAC_enabled) === "false") { + (req as any).userID = "SYSTEM"; + return next(); + } + + const token = req.get("x-user-token"); + if (!token) { + return errorHandler(401, "No token provided", req, res); + } + + const decoded = jwt.decode(token); + if (decoded && _.isObject(decoded) && decoded.roles) { + return basicToken(token, req, res, next); + } else { + return await keycloakTokenVerify(token, req, res, next); + } + } catch (error) { + return next(error); + } + }, +}; diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json new file mode 100644 index 00000000..400e218f --- /dev/null +++ b/api-service/src/middlewares/userPermissions.json @@ -0,0 +1,129 @@ +{ + "apiGroups": { + "general_access": [ + "api.datasets.list", + "api.datasets.read", + "api.datasets.export", + "api.data.out", + "api.data.exhaust", + "api.alert.list", + "api.alert.getAlertDetails", + "api.metric.list", + "api.alert.silence.list", + "api.alert.silence.get", + "api.alert.notification.list", + "api.alert.notification.get" + ], + "restricted_dataset_api": [ + "api.datasets.reset", + "api.datasets.status-transition" + ], + "alert": [ + "api.alert.create", + "api.alert.publish", + "api.alert.update", + "api.alert.delete" + ], + "metric": [ + "api.metric.add", + "api.metric.update", + "api.metric.remove" + ], + "silence": [ + "api.alert.silence.create", + "api.alert.silence.edit", + "api.alert.silence.delete" + ], + "notificationChannel": [ + "api.alert.notification.create", + "api.alert.notification.publish", + "api.alert.notification.test", + "api.alert.notification.update", + "api.alert.notification.retire" + ], + "dataset": [ + "api.datasets.create", + "api.datasets.update", + "api.datasets.import", + "api.datasets.copy", + "api.dataset.health", + "api.datasets.dataschema" + ], + "data": [ + "api.data.in" + ], + "queryTemplate": [ + "api.query.template.create", + "api.query.template.read", + "api.query.template.delete", + "api.query.template.update", + "api.query.template.query", + "api.query.template.list" + ], + "schema": [ + "api.schema.validator" + ], + "file": [ + "api.files.generate-url" + ], + "connector": [ + "api.connectors.list", + "api.connectors.read" + ], + "sqlQuery": [ + "api.obsrv.data.sql-query" + ] + }, + "roles": { + "ingestor": [ + "data" + ], + "viewer": [ + "general_access", + "connector", + "sqlQuery" + ], + "dataset_creator": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery" + ], + "dataset_manager": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "admin": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "operations_admin": [ + "alert", + "metric", + "silence", + "notificationChannel", + "general_access" + ] + } +} \ No newline at end of file diff --git a/api-service/src/models/DatasetDraft.ts b/api-service/src/models/DatasetDraft.ts index 30f56766..224e3bc7 100644 --- a/api-service/src/models/DatasetDraft.ts +++ b/api-service/src/models/DatasetDraft.ts @@ -105,7 +105,7 @@ export const DatasetDraft = sequelize.define("datasets_draft", { entry_topic: { type: DataTypes.STRING, allowNull: false - } + }, }, { timestamps: true, createdAt: "created_date", diff --git a/api-service/src/models/User.ts b/api-service/src/models/User.ts new file mode 100644 index 00000000..aa7c8d37 --- /dev/null +++ b/api-service/src/models/User.ts @@ -0,0 +1,49 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; + +export const User = sequelize.define("user", { + id: { + type: DataTypes.STRING, + primaryKey: true + }, + user_name: { + type: DataTypes.STRING + }, + password: { + type: DataTypes.STRING + }, + first_name: { + type: DataTypes.STRING + }, + last_name: { + type: DataTypes.STRING, + }, + provider: { + type: DataTypes.STRING, + }, + email_address: { + type: DataTypes.STRING, + }, + mobile_number: { + type: DataTypes.STRING, + }, + created_on: { + type: DataTypes.STRING, + }, + last_updated_on: { + type: DataTypes.STRING, + }, + roles: { + type: DataTypes.ARRAY(DataTypes.STRING), + defaultValue: ["viewer"] + }, + status: { + type: DataTypes.STRING, + defaultValue: "active" + } +}, { + tableName: "oauth_users", + timestamps: true, + createdAt: "created_date", + updatedAt: "updated_date" +}) diff --git a/api-service/src/routes/AlertsRouter.ts b/api-service/src/routes/AlertsRouter.ts index 01c843dd..6044e9fe 100644 --- a/api-service/src/routes/AlertsRouter.ts +++ b/api-service/src/routes/AlertsRouter.ts @@ -4,38 +4,39 @@ import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; import customAlertHandler from "../controllers/Alerts/Alerts"; import metricAliasHandler from "../controllers/Alerts/Metric"; import silenceHandler from "../controllers/Alerts/Silence"; +import checkRBAC from "../middlewares/RBAC_middleware"; export const alertsRouter = express.Router(); // Notifications -alertsRouter.post("/notifications/search", setDataToRequestObject("api.alert.notification.list"), notificationHandler.listHandler); -alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), notificationHandler.createHandler); -alertsRouter.get("/notifications/publish/:id", setDataToRequestObject("api.alert.notification.publish"), notificationHandler.publishHandler); -alertsRouter.post("/notifications/test", setDataToRequestObject("api.alert.notification.test"), notificationHandler.testNotifationChannelHandler); -alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), notificationHandler.updateHandler); -alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), notificationHandler.retireHandler); -alertsRouter.get("/notifications/get/:id", setDataToRequestObject("api.alert.notification.get"), notificationHandler.fetchHandler); +alertsRouter.post("/notifications/search", setDataToRequestObject("api.alert.notification.list"), checkRBAC.handler(), notificationHandler.listHandler); +alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), checkRBAC.handler(), notificationHandler.createHandler); +alertsRouter.get("/notifications/publish/:id", setDataToRequestObject("api.alert.notification.publish"), checkRBAC.handler(), notificationHandler.publishHandler); +alertsRouter.post("/notifications/test", setDataToRequestObject("api.alert.notification.test"), checkRBAC.handler(), notificationHandler.testNotifationChannelHandler); +alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), checkRBAC.handler(), notificationHandler.updateHandler); +alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), checkRBAC.handler(), notificationHandler.retireHandler); +alertsRouter.get("/notifications/get/:id", setDataToRequestObject("api.alert.notification.get"), checkRBAC.handler(), notificationHandler.fetchHandler); // alerts -alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), customAlertHandler.createAlertHandler); -alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), customAlertHandler.publishAlertHandler); -alertsRouter.post(`/search`, setDataToRequestObject("api.alert.list"), customAlertHandler.searchAlertHandler); -alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), customAlertHandler.alertDetailsHandler); -alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteAlertHandler); -alertsRouter.delete("/delete", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteSystemAlertsHandler); -alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), customAlertHandler.updateAlertHandler); +alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), checkRBAC.handler(), customAlertHandler.createAlertHandler); +alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), checkRBAC.handler(), customAlertHandler.publishAlertHandler); +alertsRouter.post(`/search`, setDataToRequestObject("api.alert.list"), checkRBAC.handler(), customAlertHandler.searchAlertHandler); +alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), checkRBAC.handler(), customAlertHandler.alertDetailsHandler); +alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), checkRBAC.handler(), customAlertHandler.deleteAlertHandler); +alertsRouter.delete("/delete", setDataToRequestObject("api.alert.delete"), checkRBAC.handler(), customAlertHandler.deleteSystemAlertsHandler); +alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), checkRBAC.handler(), customAlertHandler.updateAlertHandler); // metrics -alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), metricAliasHandler.createMetricHandler); -alertsRouter.post("/metric/alias/search", setDataToRequestObject("api.metric.list"), metricAliasHandler.listMetricsHandler); -alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"),metricAliasHandler.updateMetricHandler); -alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"),metricAliasHandler.deleteMetricHandler); -alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), metricAliasHandler.deleteMultipleMetricHandler); +alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), checkRBAC.handler(), metricAliasHandler.createMetricHandler); +alertsRouter.post("/metric/alias/search", setDataToRequestObject("api.metric.list"), checkRBAC.handler(), metricAliasHandler.listMetricsHandler); +alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"), checkRBAC.handler(), metricAliasHandler.updateMetricHandler); +alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"), checkRBAC.handler(), metricAliasHandler.deleteMetricHandler); +alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), checkRBAC.handler(), metricAliasHandler.deleteMultipleMetricHandler); // silence -alertsRouter.post("/silence/create",setDataToRequestObject("api.alert.silence.create"),silenceHandler.createHandler); -alertsRouter.get("/silence/search",setDataToRequestObject("api.alert.silence.list"),silenceHandler.listHandler); -alertsRouter.get("/silence/get/:id",setDataToRequestObject("api.alert.silence.get"),silenceHandler.fetchHandler); -alertsRouter.patch("/silence/update/:id",setDataToRequestObject("api.alert.silence.edit"),silenceHandler.updateHandler); -alertsRouter.delete("/silence/delete/:id",setDataToRequestObject("api.alert.silence.delete"),silenceHandler.deleteHandler); \ No newline at end of file +alertsRouter.post("/silence/create",setDataToRequestObject("api.alert.silence.create"), checkRBAC.handler(), silenceHandler.createHandler); +alertsRouter.get("/silence/search",setDataToRequestObject("api.alert.silence.list"), checkRBAC.handler(), silenceHandler.listHandler); +alertsRouter.get("/silence/get/:id",setDataToRequestObject("api.alert.silence.get"), checkRBAC.handler(), silenceHandler.fetchHandler); +alertsRouter.patch("/silence/update/:id",setDataToRequestObject("api.alert.silence.edit"), checkRBAC.handler(), silenceHandler.updateHandler); +alertsRouter.delete("/silence/delete/:id",setDataToRequestObject("api.alert.silence.delete"), checkRBAC.handler(), silenceHandler.deleteHandler); \ No newline at end of file diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts index ca1aec91..fdf73315 100644 --- a/api-service/src/routes/DruidProxyRouter.ts +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -9,15 +9,15 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; export const druidProxyRouter = express.Router(); // Send a 410 Gone response to all V1 API calls -druidProxyRouter.all("/datasets/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/dataset/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/datasources/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/data/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/template/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all(/\/datasets\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/dataset\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/datasources\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/data\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/template\/v1(.*)/, ResponseHandler.goneResponse); // Druid Proxy APIs for Metabase integration -druidProxyRouter.post(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative) -druidProxyRouter.get(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet) +druidProxyRouter.post(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative); +druidProxyRouter.get(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet); druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 1faaafb0..1e39d942 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -27,35 +27,40 @@ import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; -import {OperationType, telemetryAuditStart} from "../services/telemetry"; +import { OperationType, telemetryAuditStart } from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; +import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; +import checkRBAC from "../middlewares/RBAC_middleware"; +import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), dataExhaust); -router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); -router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); -router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); -router.post("/template/list", setDataToRequestObject("api.query.template.list"), listQueryTemplates); -router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); -router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); -router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); -router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); -router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), ConnectorsRead); -router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); - +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); +router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), dataOut); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(),DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), checkRBAC.handler(), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), checkRBAC.handler(), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), checkRBAC.handler(), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); +router.post("/template/create", setDataToRequestObject("api.query.template.create"), checkRBAC.handler(), createQueryTemplate); +router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), checkRBAC.handler(), readQueryTemplate); +router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), checkRBAC.handler(), deleteQueryTemplate); +router.post("/template/list", setDataToRequestObject("api.query.template.list"), checkRBAC.handler(), listQueryTemplates); +router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), checkRBAC.handler(), updateQueryTemplate); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), checkRBAC.handler(), eventValidation); +router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), checkRBAC.handler(), queryTemplate); +router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), GenerateSignedURL); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetStatusTansition); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetReset); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DataSchemaGenerator); +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetExport); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); +router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), connectorRegisterController); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); +router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) + diff --git a/api-service/src/services/CloudServices/index.ts b/api-service/src/services/CloudServices/index.ts index 6245ad19..c458016d 100644 --- a/api-service/src/services/CloudServices/index.ts +++ b/api-service/src/services/CloudServices/index.ts @@ -5,7 +5,8 @@ import { GCPStorageService } from "./GCPStorageService"; import * as _ from "lodash"; const cloudProviderName = _.get(config, "cloud_config.cloud_storage_provider"); -const cloudProviderConfig = _.get(config, "cloud_config.cloud_storage_config"); +const cloudConfig = _.get(config, "cloud_config.cloud_storage_config"); +const cloudProviderConfig = _.isString(cloudConfig) ? JSON.parse(cloudConfig) : cloudConfig; const initialiseServiceProvider = (provider: any, config: any): AzureStorageService | AWSStorageService | GCPStorageService => { switch (provider) { diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 50b42668..e8ce9a35 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -17,6 +17,7 @@ import { Datasource } from "../models/Datasource"; import { obsrvError } from "../types/ObsrvError"; import { druidHttpService } from "../connections/druidConnection"; import { tableGenerator } from "./TableGenerator"; +import { deleteAlertByDataset, deleteMetricAliasByDataset } from "./managers"; class DatasetService { @@ -92,11 +93,11 @@ class DatasetService { return responseData; } - migrateDraftDataset = async (datasetId: string, dataset: Record): Promise => { + migrateDraftDataset = async (datasetId: string, dataset: Record, userID: string): Promise => { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); + _.set(draftDataset, "updated_by", userID); const transaction = await sequelize.transaction(); - try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); @@ -125,12 +126,14 @@ class DatasetService { const transformationFields = ["field_key", "transformation_function", "mode", "metadata"] const transformations = _.includes([DatasetStatus.Live], status) ? await this.getTransformations(dataset_id, transformationFields) : await this.getDraftTransformations(dataset_id, transformationFields); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, ["field_key"]), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section) }, mode: _.get(config, ["mode"]) } @@ -147,6 +150,7 @@ class DatasetService { }) draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) draftDataset["sample_data"] = dataset_config?.mergedEvent + draftDataset["status"] = DatasetStatus.Draft return draftDataset; } @@ -157,12 +161,14 @@ class DatasetService { return "pii"; case "additionalFields": return "derived"; + case "derived": + return "derived"; default: return "transform"; } } - createDraftDatasetFromLive = async (dataset: Model) => { + createDraftDatasetFromLive = async (dataset: Model, userID: string) => { const draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); const dataset_config: any = _.get(dataset, "dataset_config"); @@ -184,12 +190,14 @@ class DatasetService { }) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, "field_key"), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, "metadata._transformedFieldDataType") || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section), }, mode: _.get(config, "mode") } @@ -198,8 +206,9 @@ class DatasetService { draftDataset["sample_data"] = dataset_config?.mergedEvent draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) } else { - const connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); - draftDataset["connectors_config"] = connectors + const v1connectors = await getV1Connectors(draftDataset.dataset_id); + const v2connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); + draftDataset["connectors_config"] = _.concat(v1connectors, v2connectors) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode"]); draftDataset["transformations_config"] = transformations } @@ -225,6 +234,7 @@ class DatasetService { draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft + draftDataset["created_by"] = userID; const result = await DatasetDraft.create(draftDataset); return _.get(result, "dataValues") } @@ -249,20 +259,27 @@ class DatasetService { } } - retireDataset = async (dataset: Record) => { + private deleteAlerts = async (dataset: any) => { + await deleteAlertByDataset(dataset); + await deleteMetricAliasByDataset(dataset); + } + + retireDataset = async (dataset: Record, updatedBy: any) => { const transaction = await sequelize.transaction(); try { - await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); - await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await Dataset.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { id: dataset.id }, transaction }); + await DatasetSourceConfig.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); + await Datasource.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); - await this.deleteDruidSupervisors(dataset); } catch (err: any) { await transaction.rollback(); throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); } + // Deleting dataset alerts and druid supervisors + await this.deleteDruidSupervisors(dataset); + await this.deleteAlerts(dataset); } findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { @@ -308,37 +325,46 @@ class DatasetService { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } - await executeCommand(draftDataset.id, "PUBLISH_DATASET"); + await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET"); } private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { + const {created_by, updated_by} = draftDataset; const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, { transaction }) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); + await DatasourceDraft.upsert(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { - const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); - const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); + const {created_by, updated_by} = draftDataset; + const allFields = await tableGenerator.getAllFieldsHudi(draftDataset, "datalake"); + const draftDatasource = this.createDraftDatasource(draftDataset, "datalake"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, { transaction }) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); + await DatasourceDraft.upsert(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { - const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); - const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); - const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") + const {created_by, updated_by} = draftDataset; + const allFields = await tableGenerator.getAllFieldsHudi(draftDataset, "datalake"); + const draftDatasource = this.createDraftDatasource(draftDataset, "datalake"); + const dsId = _.join([draftDataset.dataset_id, "events", "datalake"], "_") const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, { transaction }) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); + await DatasourceDraft.upsert(draftDatasource, { transaction }) } private createDraftDatasource = (draftDataset: Record, type: string): Record => { @@ -347,7 +373,7 @@ class DatasetService { return { id: _.join([datasource, type], "_"), datasource: draftDataset.dataset_id, - dataset_id: draftDataset.dataset_id, + dataset_id: draftDataset.id, datasource_ref: datasource, type } @@ -359,7 +385,9 @@ export const getLiveDatasetConfigs = async (dataset_id: string) => { const datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) - const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) + const connectorsV2 = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) + const connectorsV1 = await getV1Connectors(dataset_id) + const connectors = _.concat(connectorsV1, connectorsV2) if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations @@ -370,4 +398,17 @@ export const getLiveDatasetConfigs = async (dataset_id: string) => { return datasetRecord; } +export const getV1Connectors = async (datasetId: string) => { + const v1connectors = await datasetService.getConnectorsV1(datasetId, ["id", "connector_type", "connector_config"]); + const modifiedV1Connectors = _.map(v1connectors, (config) => { + return { + id: _.get(config, "id"), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), + version: "v1" + } + }) + return modifiedV1Connectors; +} + export const datasetService = new DatasetService(); \ No newline at end of file diff --git a/api-service/src/services/SchemaGenerateService/DataSchemaService.ts b/api-service/src/services/SchemaGenerateService/DataSchemaService.ts index 67bde9e7..aec2f8d3 100644 --- a/api-service/src/services/SchemaGenerateService/DataSchemaService.ts +++ b/api-service/src/services/SchemaGenerateService/DataSchemaService.ts @@ -5,17 +5,17 @@ import moment from "moment"; import { SchemaGenerationException } from "../../exceptions/SchemaGenerationException"; const DATE_FORMATS = [ - "MM/DD/YYYY","DD/MM/YYYY", "YYYY-MM-DD", "YYYY-DD-MM", "YYYY/MM/DD", + "MM/DD/YYYY", "DD/MM/YYYY", "YYYY-MM-DD", "YYYY-DD-MM", "YYYY/MM/DD", "DD-MM-YYYY", "MM-DD-YYYY", "MM-DD-YYYY HH:mm:ss", "YYYY/MM/DD HH:mm:ss", - "YYYY-MM-DD HH:mm:ss", "YYYY-DD-MM HH:mm:ss", "DD/MM/YYYY HH:mm:ss", - "DD-MM-YYYY HH:mm:ss", "MM-DD-YYYY HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss.SSS", - "YYYY-DD-MM HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss.SSS", "DD/MM/YYYY HH:mm:ss.SSS", - "DD-MM-YYYY HH:mm:ss.SSS", "DD-MM-YYYYTHH:mm:ss.SSSZ", "YYYY-MM-DDTHH:mm:ss.SSSZ", + "YYYY-MM-DD HH:mm:ss", "YYYY-DD-MM HH:mm:ss", "DD/MM/YYYY HH:mm:ss", + "DD-MM-YYYY HH:mm:ss", "MM-DD-YYYY HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss.SSS", + "YYYY-DD-MM HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss.SSS", "DD/MM/YYYY HH:mm:ss.SSS", + "DD-MM-YYYY HH:mm:ss.SSS", "DD-MM-YYYYTHH:mm:ss.SSSZ", "YYYY-MM-DDTHH:mm:ss.SSSZ", "YYYY-DD-MMTHH:mm:ss.SSSZ", "YYYY/MM/DDTHH:mm:ss.SSSZ", "DD/MM/YYYYTHH:mm:ss.SSSZ", "YYYY-DD-MMTHH:mm:ss.SSS", "YYYY/MM/DDTHH:mm:ss.SSS", "DD/MM/YYYYTHH:mm:ss.SSS", "MM-DD-YYYYTHH:mm:ss.SSSZ", "DD-MM-YYYYTHH:mm:ssZ", "YYYY-MM-DDTHH:mm:ssZ", - "YYYY-DD-MMTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ssZ", "DD/MM/YYYYTHH:mm:ssZ", "MM-DD-YYYYTHH:mm:ssZ", - "MM-DD-YYYYTHH:mm:ss", "DD-MM-YYYYTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss", "YYYY-DD-MMTHH:mm:ss", + "YYYY-DD-MMTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ssZ", "DD/MM/YYYYTHH:mm:ssZ", "MM-DD-YYYYTHH:mm:ssZ", + "MM-DD-YYYYTHH:mm:ss", "DD-MM-YYYYTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss", "YYYY-DD-MMTHH:mm:ss", "YYYY/MM/DDTHH:mm:ss", "DD/MM/YYYYTHH:mm:ss", "DD-MM-YYYY HH:mm:ss.SSSZ", "YYYY-MM-DD HH:mm:ss.SSSZ", "YYYY-DD-MM HH:mm:ss.SSSZ", "YYYY/MM/DD HH:mm:ss.SSSZ", "DD/MM/YYYY HH:mm:ss.SSSZ", "MM-DD-YYYY HH:mm:ss.SSSZ", "DD-MM-YYYY HH:mm:ssZ", "YYYY-MM-DD HH:mm:ssZ", "YYYY-DD-MM HH:mm:ssZ", @@ -38,16 +38,30 @@ const DATE_FORMATS = [ export class SchemaInference { public inferSchema(sample: any) { - const schema = _.map(sample, (value): any => this.validateEpoch(inferSchema(value).toJSONSchema({ includeSchema: true }), value, "properties")) - return schema + if (!Array.isArray(sample)) { + throw new SchemaGenerationException("Invalid input: sample must be an array.", httpStatus.BAD_REQUEST); + } + const removedAllKeys: any[] = [] + const schema = _.map(sample, (value): any => { + const { cleanedData, removedKeys } = this.removeEmpty(value) + removedAllKeys.push(...removedKeys) + return this.validateEpoch(inferSchema(cleanedData).toJSONSchema({ includeSchema: true }), cleanedData, "properties") + }) + return { schema, removedKeys: removedAllKeys } } public inferBatchSchema(sample: Map[], extractionKey: string) { - return _.flatMap(sample, (value) => { + if (!Array.isArray(sample)) { + throw new SchemaGenerationException("Invalid input: sample must be an array.", httpStatus.BAD_REQUEST); + } + const removedAllKeys: any[] = [] + const schema = _.flatMap(sample, (value) => { if (extractionKey) { const extracted = _.get(value, extractionKey); if (extracted) { - return this.inferSchema(extracted); + const { schema, removedKeys } = this.inferSchema(extracted); + removedAllKeys.push(...removedKeys) + return schema } else { throw new SchemaGenerationException("Unable to extract the batch data.", httpStatus.BAD_REQUEST); } @@ -55,6 +69,22 @@ export class SchemaInference { throw new SchemaGenerationException("Extraction key not found.", httpStatus.BAD_REQUEST); } }) + return { schema, removedKeys: removedAllKeys } + } + // Only removes empty object and array at all the levels + private removeEmpty(data: any, parentKey = "", removedKeys: any[] = []) { + Object.keys(data).forEach((key) => { + const value = data[key]; + const fullKey = parentKey ? `${parentKey}.${key}` : key; + if (typeof value === "object" && value !== null) { + this.removeEmpty(value, fullKey, removedKeys); + if (_.isEmpty(value)) { + delete data[key]; + removedKeys.push({ "key": fullKey, value }); + } + } + }); + return { cleanedData: data, removedKeys }; } private validateEpoch(schema: any, sample: any, path: any) { @@ -67,7 +97,7 @@ export class SchemaInference { if (isValidTimestamp) { _.set(schema, `${path}.${key}.format`, type) } - else if(format && ["date-time", "time", "date"].includes(format) && !isValidTimestamp) { + else if (format && ["date-time", "time", "date"].includes(format) && !isValidTimestamp) { _.unset(schema, `${path}.${key}.format`) } }); @@ -79,7 +109,7 @@ export class SchemaInference { const epochRegex = /^\d+$/ig; switch (dataType) { case "string": - if(epochRegex.test(value)){ + if (epochRegex.test(value)) { const parsedValue = parseInt(value, 10); // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds return { @@ -93,12 +123,12 @@ export class SchemaInference { case "number": // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds return { - isValidTimestamp: value >= 946684800 && moment(value).isValid(), + isValidTimestamp: value >= 946684800 && moment(value).isValid(), type: "epoch" }; default: return { - isValidTimestamp: false, + isValidTimestamp: false, type: "" }; } diff --git a/api-service/src/services/SchemaGenerateService/SchemaMapping.json b/api-service/src/services/SchemaGenerateService/SchemaMapping.json index 19843cf5..e9fe2a6f 100644 --- a/api-service/src/services/SchemaGenerateService/SchemaMapping.json +++ b/api-service/src/services/SchemaGenerateService/SchemaMapping.json @@ -66,10 +66,6 @@ "epoch":{ "jsonSchema": "integer", "datasource": "long" - }, - "number": { - "jsonSchema": "number", - "datasource": "double" } } }, diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index eb3065ae..7030ad95 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -15,7 +15,7 @@ class BaseTableGenerator { const properties: Record[] = [] const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { _.mapKeys(schema, function (value, parentKey) { - const newKey = (prev) ? _.join([prev, parentKey], ".") : parentKey; + const newKey = (prev) ? type === "druid" ? _.join([prev, parentKey], ".") : _.replace(_.join([prev, parentKey], "_"), /\./g, "_") : parentKey; const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], "") : _.join(["$.['", parentKey, "']"], ""); switch (value["type"]) { case "object": @@ -24,7 +24,7 @@ class BaseTableGenerator { case "array": if (type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { _.mapKeys(_.get(value, "items.properties"), function (value, childKey) { - const objChildKey = _.join([newKey, childKey], ".") + const objChildKey = type === "druid" ? _.join([newKey, childKey], ".") : _.replace(_.join([prev, childKey], "_"), /\./g, "_") properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), { expr: _.join([newExpr, "[*].['", childKey, "']"], ""), name: objChildKey, data_type: "array" })) }) } else { @@ -79,6 +79,38 @@ class BaseTableGenerator { _.remove(dataFields, { is_deleted: true }) // Delete all the excluded fields return dataFields; } + + getAllFieldsHudi = async (dataset: Record, type: string): Promise[]> => { + + const { data_schema, denorm_config, transformations_config } = dataset + let dataFields = this.flattenSchema(data_schema, type); + if (!_.isEmpty(denorm_config.denorm_fields)) { + for (const denormField of denorm_config.denorm_fields) { + const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); + const properties = this.flattenSchema(denormDataset.data_schema, type); + const transformProps = _.map(properties, (prop) => { + _.set(prop, "name", _.join([_.replace(denormField.denorm_out_field, /\./g, "_"), prop.name], "_")); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + return prop; + }); + dataFields.push(...transformProps); + } + } + if (!_.isEmpty(transformations_config)) { + const transformationFields = _.map(transformations_config, (tf) => ({ + expr: "$." + tf.field_key, + name: _.replace(tf.field_key, /\./g, "_"), + data_type: tf.transformation_function.datatype, + arrival_format: tf.transformation_function.datatype, + type: tf.transformation_function.datatype + })) + const originalFields = _.differenceBy(dataFields, transformationFields, "name") + dataFields = _.concat(originalFields, transformationFields) + } + dataFields.push(rawIngestionSpecDefaults.hudiSynctsField) + _.remove(dataFields, { is_deleted: true }) // Delete all the excluded fields + return dataFields; + } } class TableGenerator extends BaseTableGenerator { @@ -86,18 +118,19 @@ class TableGenerator extends BaseTableGenerator { getDruidIngestionSpec = (dataset: Record, allFields: Record[], datasourceRef: string) => { const { dataset_config, router_config } = dataset + const ingestionSpecDefaults = _.cloneDeep(rawIngestionSpecDefaults) return { "type": "kafka", "spec": { "dataSchema": { "dataSource": datasourceRef, - "dimensionsSpec": { "dimensions": this.getDruidDimensions(allFields, this.getTimestampKey(dataset), dataset_config.keys_config.partition_key) }, - "timestampSpec": { "column": this.getTimestampKey(dataset), "format": "auto" }, + "dimensionsSpec": { "dimensions": this.getDruidDimensions(allFields, this.getTimestampKey(dataset, "druid"), dataset_config.keys_config.partition_key) }, + "timestampSpec": { "column": this.getTimestampKey(dataset, "druid"), "format": "auto" }, "metricsSpec": [], - "granularitySpec": rawIngestionSpecDefaults.granularitySpec + "granularitySpec": ingestionSpecDefaults.granularitySpec }, - "tuningConfig": rawIngestionSpecDefaults.tuningConfig, - "ioConfig": _.merge(rawIngestionSpecDefaults.ioConfig, { + "tuningConfig": ingestionSpecDefaults.tuningConfig, + "ioConfig": _.merge(ingestionSpecDefaults.ioConfig, { "topic": router_config.topic, "inputFormat": { "flattenSpec": { @@ -142,28 +175,28 @@ class TableGenerator extends BaseTableGenerator { } } - private getDruidFlattenSpec = (allFields: Record) => { - return _.union( - _.map(allFields, (field) => { - return { - type: "path", - expr: field.expr, - name: field.name - } - }), - rawIngestionSpecDefaults.flattenSpec - ) + private getDruidFlattenSpec = (fields: Record) => { + const allfields = _.map(fields, (field) => { + return { + type: "path", + expr: field.expr, + name: field.name + } + }); + return _.uniqBy([...allfields, ...rawIngestionSpecDefaults.flattenSpec], "name") } getHudiIngestionSpecForCreate = (dataset: Record, allFields: Record[], datasourceRef: string) => { const primaryKey = this.getPrimaryKey(dataset); const partitionKey = this.getHudiPartitionKey(dataset); - const timestampKey = this.getTimestampKey(dataset); + const timestampKey = this.getTimestampKey(dataset, "datalake"); return { dataset: dataset.dataset_id, schema: { - table: datasourceRef, + table: _.includes(datasourceRef, "-") + ? _.replace(datasourceRef, /-/g, "_") + : datasourceRef, partitionColumn: partitionKey, timestampColumn: timestampKey, primaryKey: primaryKey, @@ -199,12 +232,10 @@ class TableGenerator extends BaseTableGenerator { return newHudiSpec; } + // eslint-disable-next-line private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string): Record[] => { const dataFields = _.cloneDeep(allFields); - _.remove(dataFields, { name: primaryKey }) - _.remove(dataFields, { name: partitionKey }) - _.remove(dataFields, { name: timestampKey }) let index = 1; const transformFields = _.map(dataFields, (field) => { return { @@ -213,7 +244,7 @@ class TableGenerator extends BaseTableGenerator { "index": index++ } }) - _.each(rawIngestionSpecDefaults.dimensions, (field) => { + _.each(rawIngestionSpecDefaults.hudi_dimensions, (field) => { transformFields.push({ "type": field.type, "name": field.name, @@ -280,20 +311,25 @@ class TableGenerator extends BaseTableGenerator { name: field.name } }), - rawIngestionSpecDefaults.flattenSpec + rawIngestionSpecDefaults.hudi_flattenSpec ) } private getPrimaryKey = (dataset: Record): string => { - return dataset.dataset_config.keys_config.data_key; + return _.replace(dataset.dataset_config.keys_config.data_key, /\./g, "_"); } private getHudiPartitionKey = (dataset: Record): string => { - return dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; + const partitionKey = dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; + return _.replace(partitionKey, /\./g, "_") } - private getTimestampKey = (dataset: Record): string => { - return dataset.dataset_config.keys_config.timestamp_key; + private getTimestampKey = (dataset: Record, type: string): string => { + const timestamp = dataset.dataset_config.keys_config.timestamp_key; + if (type === "druid") { + return timestamp; + } + return _.replace(timestamp, /\./g, "_"); } } diff --git a/api-service/src/services/UserService.ts b/api-service/src/services/UserService.ts new file mode 100644 index 00000000..6850cb91 --- /dev/null +++ b/api-service/src/services/UserService.ts @@ -0,0 +1,11 @@ +import { User } from "../models/User"; + +class UserService { + + getUser = (where?: Record, attributes?: string[]): Promise => { + return User.findOne({ where, attributes, raw: true }); + } + +} + +export const userService = new UserService(); \ No newline at end of file diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index 84b666d9..585b32b7 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -24,16 +24,16 @@ const getService = (manager: string) => { }; export const publishAlert = async (payload: Record) => { - const { id, manager } = payload; + const { id, manager, updated_by } = payload; const service = getService(manager); const publishResponse = await service.publishAlert(payload) - await updateStatus(id, "live"); + await updateStatus(id, "live", updated_by); return publishResponse; }; -const updateStatus = (id: string, status: string) => { - return Alert.update({ status }, { where: { id } }); +const updateStatus = (id: string, status: string, updated_by: string) => { + return Alert.update({ status, updated_by }, { where: { id } }); } const deleteRule = (id: string) => { @@ -41,7 +41,7 @@ const deleteRule = (id: string) => { } export const deleteAlertRule = async (payload: Record, hardDelete: boolean) => { - const { id, manager, status } = payload; + const { id, manager, status, updated_by } = payload; if (status == "live") { try { @@ -56,7 +56,7 @@ export const deleteAlertRule = async (payload: Record, hardDelete: return deleteRule(id); } - return updateStatus(id, "retired"); + return updateStatus(id, "retired", updated_by); } @@ -138,11 +138,7 @@ export const deleteAlertByDataset = async (payload: Record) => { export const deleteMetricAliasByDataset = async (payload: Record) => { try { const { name } = payload; - const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name } }) - if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) - for (const payload of metricAliasPayload) { - await payload.destroy() - } + await Metrics.destroy({ where: { component: "datasets", subComponent: name } }) return constants.METRIC_ALIAS_DELETED_SUCCESSFULLY; } catch (error: any) { throw new Error(constants.METRIC_ALIAS_NOT_DELETED); diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index d4eca61b..8408dfb3 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -10,15 +10,15 @@ const telemetryTopic = _.get(appConfig, "telemetry_dataset"); export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } -const getDefaults = () => { +const getDefaults = (userID:any) => { return { eid: "AUDIT", ets: Date.now(), ver: "1.0.0", mid: v4(), actor: { - id: "SYSTEM", - type: "User" + id: userID || "SYSTEM", + type: "User", }, context: { env, @@ -128,7 +128,7 @@ export const processAuditEvents = (request: Request) => { _.set(auditEvent, "edata.transition.toState", toState); _.set(auditEvent, "edata.transition.fromState", fromState); } - const telemetryEvent = getDefaults(); + const telemetryEvent = getDefaults((request as any)?.userID); _.set(telemetryEvent, "edata", edata); _.set(telemetryEvent, "object", { ...(object.id && object.type && { ...object, ver: "1.0.0" }) }); sendTelemetryEvents(telemetryEvent); diff --git a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts index 487cd77e..1ff1c9b4 100644 --- a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts @@ -36,15 +36,13 @@ describe("DATA INGEST API", () => { it("it should ingest data for individual event", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: "local.test.topic", - }, - extraction_config: { - is_batch_event: false, - extraction_key: "events", - batch_id: "id" - } + dataset_config: { + entry_topic: "local.test.topic", + }, + extraction_config: { + is_batch_event: false, + extraction_key: "events", + batch_id: "id" } }) }) @@ -71,10 +69,8 @@ describe("DATA INGEST API", () => { it("it should ingest data successfully", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: "local.test.topic", - } + dataset_config: { + entry_topic: "local.test.topic", } }) }) @@ -97,13 +93,37 @@ describe("DATA INGEST API", () => { }) }); + it("it should ingest data successfully v2", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ + api_version: "v2", + entry_topic: "local.test.topic", + }) + }) + const connectionStub = sinon.stub(kafkaModule, "connect").returns(true); + const sendStub = sinon.stub(kafkaModule, "send").returns(resultResponse); + chai + .request(app) + .post(apiEndpoint) + .send(TestInputsForDataIngestion.SAMPLE_INPUT_1) + .end((err, res) => { + res.should.have.status(200); + res.body.should.be.a("object"); + res.body.should.have.property("result"); + res.body.id.should.be.eq("api.data.in"); + res.body.result.message.should.be.eq("Data ingested successfully") + connectionStub.restore() + sendStub.restore() + chai.spy.restore(Dataset, "findOne") + done() + }) + }); + it("Failed to connect kafka.", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: "local.test.topic", - } + dataset_config: { + entry_topic: "local.test.topic", } }) }) @@ -125,9 +145,7 @@ describe("DATA INGEST API", () => { it("Entry topic not found", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: {} - } + dataset_config: {} }) }) @@ -195,8 +213,7 @@ describe("DATA INGEST API", () => { res.body.should.be.a("object") res.body.id.should.be.eq("api.data.in"); res.body.params.status.should.be.eq("FAILED"); - res.body.error.code.should.be.eq("DATA_INGESTION_FAILED"); - res.body.error.message.should.be.eq("Failed to ingest data") + res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); }); diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 96d53e38..ea905305 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -7,6 +7,7 @@ import { config } from "../../../configs/Config"; import chaiSpies from "chai-spies" import { describe, it } from "mocha"; import { Datasource } from "../../../models/Datasource"; +import { druidHttpService } from "../../../connections/druidConnection"; chai.use(chaiSpies) chai.should(); chai.use(chaiHttp); @@ -18,6 +19,7 @@ const nativeQueryEndpointDruid = config?.query_api?.druid?.native_query_path; const sqlQueryEndpoint = config?.query_api?.druid?.sql_query_path; const response = [{ dataValues: { datasource_ref: "test.1_rollup_week", metadata: { aggregated: true, granularity: "week" } } }] +const invalidResponse = [{ dataValues: { datasource_ref: "test.1_rollup_week", metadata: { aggregated: true, granularity: "n/a" } } }] const msgid = "e180ecac-8f41-4f21-9a21-0b3a1a368917"; describe("QUERY API TESTS", () => { @@ -33,6 +35,11 @@ describe("QUERY API TESTS", () => { response ) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["telemetry-events.1_rollup"]) @@ -46,7 +53,7 @@ describe("QUERY API TESTS", () => { res.body.params.msgid.should.be.eq(msgid); res.body.responseCode.should.be.eq("NOT_FOUND"); res.body.error.message.should.be.eq("Dataset telemetry-events with table week is not available for querying"); - res.body.error.code.should.be.eq("DATA_OUT_SOURCE_NOT_FOUND"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FOUND"); done(); }); }); @@ -65,7 +72,80 @@ describe("QUERY API TESTS", () => { res.body.responseCode.should.be.eq("NOT_FOUND"); res.body.params.msgid.should.be.eq(msgid); res.body.error.message.should.be.eq("Datasource telemetry-events not available for querying"); - res.body.error.code.should.be.eq("DATA_OUT_SOURCE_NOT_FOUND"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FOUND"); + done(); + }); + }); + + it("Query api failure: Datasource not available in druid", (done) => { + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve( + response + ) + }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "telemetry_events": 100 } + }) + }) + chai + .request(app) + .post("/v2/data/query/telemetry-events") + .send(JSON.parse(TestQueries.VALID_QUERY)) + .end((err, res) => { + res.should.have.status(404); + res.body.params.status.should.be.eq("FAILED"); + res.body.responseCode.should.be.eq("NOT_FOUND"); + res.body.params.msgid.should.be.eq(msgid); + res.body.error.message.should.be.eq("Datasource not available for querying"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_AVAILABLE"); + done(); + }); + }); + + it("Query api failure: Datasource not fully loaded in druid", (done) => { + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve( + response + ) + }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 20 } + }) + }) + chai + .request(app) + .post("/v2/data/query/telemetry-events") + .send(JSON.parse(TestQueries.VALID_QUERY)) + .end((err, res) => { + res.should.have.status(416); + res.body.params.status.should.be.eq("FAILED"); + res.body.responseCode.should.be.eq("RANGE_NOT_SATISFIABLE"); + res.body.params.msgid.should.be.eq(msgid); + res.body.error.message.should.be.eq("Datasource not fully available for querying"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FULLY_AVAILABLE"); + done(); + }); + }); + + it("Query api failure: Datasource not found", (done) => { + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve( + invalidResponse + ) + }) + chai + .request(app) + .post("/v2/data/query/telemetry-events") + .send(JSON.parse(TestQueries.VALID_QUERY)) + .end((err, res) => { + res.should.have.status(404); + res.body.params.status.should.be.eq("FAILED"); + res.body.responseCode.should.be.eq("NOT_FOUND"); + res.body.params.msgid.should.be.eq(msgid); + res.body.error.message.should.be.eq("Datasource not found to query"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FOUND"); done(); }); }); @@ -74,6 +154,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -89,8 +174,8 @@ describe("QUERY API TESTS", () => { res.body.params.status.should.be.eq("FAILED"); res.body.params.msgid.should.be.eq(msgid); res.body.params.should.have.property("resmsgid"); - res.body.error.message.should.be.eq("Unable to process the query."); - res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); + res.body.error.message.should.be.eq("Request failed with status code 500"); + res.body.error.code.should.be.eq("ERR_BAD_RESPONSE"); res.body.responseCode.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); @@ -100,6 +185,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -115,8 +205,8 @@ describe("QUERY API TESTS", () => { res.body.params.status.should.be.eq("FAILED"); res.body.params.msgid.should.be.eq(msgid); res.body.params.should.have.property("resmsgid"); - res.body.error.message.should.be.eq("Unable to process the query."); - res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); + res.body.error.message.should.be.eq("Request failed with status code 500"); + res.body.error.code.should.be.eq("ERR_BAD_RESPONSE"); res.body.responseCode.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); @@ -126,6 +216,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -153,6 +248,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -180,6 +280,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -223,6 +328,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index 51953ba0..fd1aabdd 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -114,6 +114,4 @@ describe("DATASET CREATE API", () => { done(); }); }); - - }) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index cfd333eb..c3e766e7 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -31,9 +31,18 @@ describe("DATASET READ API", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "name": "sb-telemetry", "version": 1 }) }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(ConnectorInstances, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?fields=name,version") + .get("/v2/datasets/read/sb-telemetry?fields=name,version,connectors_config,transformations_config") .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -42,7 +51,7 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ name: "sb-telemetry", version: 1 })) + result.should.be.eq(JSON.stringify({ "name": "sb-telemetry", "version": 1, "connectors_config": [], "transformations_config": [] })) done(); }); }); @@ -69,6 +78,15 @@ describe("DATASET READ API", () => { }); it("Dataset read success: Fetch live dataset when mode param not provided", (done) => { + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) + }) + chai.spy.on(ConnectorInstances, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) chai.spy.on(Dataset, "findOne", () => { return Promise.resolve(TestInputsForDatasetRead.LIVE_SCHEMA) }) @@ -83,7 +101,7 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.status.should.be.eq("Live") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA })) + result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA, connectors_config: TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2, transformations_config: TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA })) done(); }); }); @@ -98,6 +116,9 @@ describe("DATASET READ API", () => { chai.spy.on(DatasetTransformations, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) chai.spy.on(ConnectorInstances, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) }) @@ -267,7 +288,7 @@ describe("DATASET READ API", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) }) chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{"dataset_id":"master_dataset", "dataset_config":{"cache_config":{"redis_db":20}}}]) + return Promise.resolve([{ "dataset_id": "master_dataset", "dataset_config": { "cache_config": { "redis_db": 20 } } }]) }) chai .request(app) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index e26ddafe..38dced09 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -38,7 +38,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { @@ -80,7 +80,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { @@ -122,7 +122,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "api_version":"v2", "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) chai.spy.on(Datasource, "findOne", () => { @@ -289,7 +289,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index 1a0428af..e09dcc41 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -14,6 +14,7 @@ import { Datasource } from "../../../models/Datasource"; import { commandHttpService } from "../../../connections/commandServiceConnection"; import { druidHttpService } from "../../../connections/druidConnection"; import { sequelize } from "../../../connections/databaseConnection"; +import { datasetService } from "../../../services/DatasetService"; chai.use(spies); chai.should(); @@ -46,6 +47,9 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve([{ datasource_ref: "telemetry" }]) }) + chai.spy.on(datasetService, "deleteAlerts", () => { + return Promise.resolve({}) + }) chai.spy.on(druidHttpService, "post", () => { return Promise.resolve({}) }) @@ -91,6 +95,9 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Dataset, "update", () => { return Promise.resolve({}) }) + chai.spy.on(datasetService, "deleteAlerts", () => { + return Promise.resolve({}) + }) chai.spy.on(commandHttpService, "post", () => { return Promise.resolve({}) }) @@ -130,6 +137,9 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Datasource, "update", () => { return Promise.resolve({}) }) + chai.spy.on(datasetService, "deleteAlerts", () => { + return Promise.resolve({}) + }) chai.spy.on(Dataset, "update", () => { return Promise.resolve({}) }) diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index c2050f23..7edad64e 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -53,7 +53,9 @@ describe("DATASET UPDATE API", () => { id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id" : "master-telemetry", + "redis_db": 10 }] }, api_version: "v2" }) @@ -80,7 +82,6 @@ describe("DATASET UPDATE API", () => { }); it("Dataset updation failure: When no fields with dataset_id is provided in the request payload", (done) => { - chai .request(app) .patch("/v2/datasets/update") @@ -101,7 +102,6 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) - chai .request(app) .patch("/v2/datasets/update") @@ -231,7 +231,6 @@ describe("DATASET UPDATE API", () => { }); it("Failure: Failed to update the dataset name", (done) => { - chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index ae9628a4..bd103062 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -191,7 +191,7 @@ export const TestInputsForDatasetUpdate = { "dataset_config": { "data_key": "eid", "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "files_upload_path": ["/config/file.json"] } } }, @@ -233,7 +233,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key1", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -340,14 +344,16 @@ export const TestInputsForDatasetUpdate = { { "values": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id" : "master-telemetry" }, "action": "upsert" }, { "values": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id" : "master-telemetry" }, "action": "upsert" } diff --git a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts index 8da1818b..55368903 100644 --- a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts +++ b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts @@ -8,6 +8,7 @@ import { Datasource } from "../../../models/Datasource"; import nock from "nock"; import { config } from "../../../configs/Config"; import { templateQueryApiFixtures } from "./Fixtures"; +import { druidHttpService } from "../../../connections/druidConnection"; const apiId = "api.query.template.query"; const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" @@ -50,6 +51,11 @@ describe("QUERY TEMPLATE API", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_month": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_month"]) @@ -93,6 +99,11 @@ describe("QUERY TEMPLATE API", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_month": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_month"]) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 8814daac..d4cd25b5 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -93,6 +93,7 @@ paths: dataset_config: data_key: mid timestamp_key: ets + files_upload_path: ["telemetry.json"] tags: - tag1 - tag2 @@ -261,10 +262,12 @@ paths: - values: denorm_key: eid denorm_out_field: userdata + dataset_id: master-telemetry action: remove - values: denorm_key: eid denorm_out_field: edata + dataset_id: trip-record action: add transformation_config: - values: diff --git a/command-service/Dockerfile b/command-service/Dockerfile index 2ca4edef..4e7fe11a 100644 --- a/command-service/Dockerfile +++ b/command-service/Dockerfile @@ -1,18 +1,19 @@ -FROM --platform=linux/amd64 python:3.12-alpine -COPY --from=ubuntu /usr/local/bin /usr/local/bin +FROM --platform=linux/amd64 python:3.12-slim -RUN apk update && apk add curl jq vim && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + git \ + librdkafka-dev \ + curl \ + jq \ + vim \ + && rm -rf /var/lib/apt/lists/* -RUN cd /tmp && curl -OL https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz \ - && tar -zxvf helm-v3.13.2-linux-amd64.tar.gz \ - && mv linux-amd64/helm /usr/local/bin/helm +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ + chmod +x kubectl && \ + mv kubectl /usr/local/bin/ -# RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ -# && mv kubectl /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl - -# RUN apk add openssl==3.3.0-r2 libcrypto3==3.3.0-r2 || \ -RUN apk add openssl libcrypto3 -RUN apk upgrade --no-cache +RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 3da5d6ce..74fc9cca 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -49,7 +49,7 @@ spec: containers: - name: {{ $jobName }}-taskmanager image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} workingDir: {{ .Values.taskmanager.flink_work_dir }} args: ["taskmanager"] env: @@ -69,33 +69,14 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - # - name: flink-config-volume - # mountPath: /data/flink/conf/connectors-scala-config.conf - # subPath: connectors-scala-config.conf - # - name: flink-config-volume - # mountPath: /data/flink/conf/connectors-python-config.conf - # subPath: connectors-python-config.yaml - # - name: flink-config-volume - # mountPath: /opt/flink/conf/flink-conf.yaml - # subPath: flink-conf.yaml - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties volumes: - # - name: flink-config-volume - # configMap: - # name: flink-connector-conf - # items: - # - key: connectors-scala-config.conf - # path: connectors-scala-config.conf - # - key: connectors-python-config.yaml - # path: connectors-python-config.yaml - name: flink-common-volume configMap: name: {{ $jobName }}-config items: - # - key: flink-conf - # path: flink-conf.yaml - key: log4j_console_properties path: log4j-console.properties {{ $component := "jobmanager" }} @@ -143,7 +124,7 @@ spec: containers: - name: {{ $jobName }}-jobmanager image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} workingDir: /opt/flink # command: ["/bin/sh", "-c"] args: ["jobmanager"] @@ -157,35 +138,40 @@ spec: metrics.reporter.prom.host: {{ $jobName }}-jobmanager metrics.reporter.prom.port: 9250 volumeMounts: - - name: flink-config-volume + - name: connector-config-volume mountPath: /data/flink/conf/connectors-scala-config.conf subPath: connectors-scala-config.conf - - name: flink-config-volume - mountPath: /data/flink/conf/connectors-python-config.conf - subPath: connectors-python-config.conf + - name: connector-config-volume + mountPath: /data/flink/conf/connectors-python-config.yaml + subPath: connectors-python-config.yaml - name: data - mountPath: /flink/connectors + mountPath: /data/connectors - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties - name: {{ $jobName }}-job-submit image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} workingDir: /opt/flink env: - name: CONNECTOR_ID value: {{ index $jobData "connector_id" }} + - name: CONFIG_PATH + value: /data/flink/conf/ + - name: CONFIG_FILE + value: connectors-python-config.yaml command: - /bin/bash - -c - | - /usr/bin/python3.11 /data/flink/connectors/connectors-init/connector.py; + /usr/bin/python3 /data/connectors-init/connector.py; sleep 30s; - /opt/flink/bin/flink run -m \ + /opt/flink/bin/flink run -d -m \ {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ - /flink/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ + /data/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ --config.file.path /data/flink/conf/connectors-scala-config.conf \ + --metadata.id {{ index $jobData "connector_id" }} \ {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ {{- end }} @@ -198,7 +184,7 @@ spec: {{- if eq .Values.checkpoint_store_type "gcs" }} "-Dgoogle.cloud.auth.service.account.enable=true" \ {{- end }} - ; + ; sleep 10s; echo "Job submitted"; tail -f /dev/null; ports: {{- range .Values.service.ports }} - name: {{ .name }} @@ -208,23 +194,21 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - - name: flink-config-volume + - name: connector-config-volume mountPath: /data/flink/conf/connectors-scala-config.conf subPath: connectors-scala-config.conf - - name: flink-config-volume + - name: connector-config-volume mountPath: /data/flink/conf/connectors-python-config.yaml subPath: connectors-python-config.yaml - name: data - mountPath: /flink/connectors + mountPath: /data/connectors - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties - - volumes: - - name: flink-config-volume + - name: connector-config-volume configMap: - name: flink-connector-conf + name: connectors-config items: - key: connectors-scala-config.conf path: connectors-scala-config.conf @@ -248,7 +232,6 @@ spec: resources: requests: storage: 2Gi - storageClassName: "standard" {{- end }} {{- end}} diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index ff505a84..ed51316d 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -18,6 +18,7 @@ registry: sanketikahub repository: flink-connectors tag: 1.17.2-scala_2.12-java11 imagePullSecrets: [] +imagePullPolicy: IfNotPresent ## Databases global: @@ -205,65 +206,6 @@ log4j_console_properties: | logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline logger.netty.level = OFF -baseconfig: | - kafka { - broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" - producer.broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" - consumer.broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" - zookeeper = "{{ .Values.global.zookeeper.host }}:{{ .Values.global.zookeeper.port }}" - producer { - max-request-size = 1572864 - batch.size = 98304 - linger.ms = 10 - compression = "snappy" - } - output.system.event.topic = ${job.env}".system.events" - - output.failed.topic = ${job.env}".failed" - } - job { - env = "{{ .Values.global.env }}" - enable.distributed.checkpointing = {{ .Values.checkpointing.enabled }} - statebackend { - base.url = "{{ .Values.checkpointing.statebackend }}" - } - } - - task { - parallelism = 1 - consumer.parallelism = 1 - checkpointing.interval = 10000 - checkpointing.pause.between.seconds = 10000 - restart-strategy.attempts = 3 - restart-strategy.delay = 30000 # in milli-seconds - } - - redis.connection.timeout = 100 - redis { - host = "{{ .Values.global.redis_dedup.host }}" - port = {{ .Values.global.redis_dedup.port }} - } - - redis-meta { - host = "{{ .Values.global.redis_denorm.host }}" - port = {{ .Values.global.redis_denorm.port }} - } - - postgres { - host = "{{ .Values.global.postgresql.host }}" - port = "{{ .Values.global.postgresql.port }}" - maxConnections = "{{ .Values.postgres.max_connections }}" - sslMode = "{{ .Values.postgres.sslmode }}" - user = "{{ .Values.global.postgresql.obsrv.user }}" - password = "{{ .Values.global.postgresql.obsrv.password }}" - database = "{{ .Values.global.postgresql.obsrv.name }}" - } - - lms-cassandra { - host = "{{ .Values.global.cassandra.host }}" - port = "{{ .Values.global.cassandra.port }}" - } - flink_resources: taskmanager: resources: @@ -300,8 +242,10 @@ serviceMonitor: jobLabel: "app.kubernetes.io/name" port: prom -flink_jobs: - + +# override flink_jobs +# flink_jobs: + commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl b/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl index f4b82f3c..094968a5 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl +++ b/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl @@ -1,4 +1,4 @@ {{- define "base.cronReleaseName" -}} - {{- $name := printf "%s--%s" .Chart.Name .Values.instance_id }} + {{- $name := printf "%s" .Values.instance_id }} {{- default .Values.instance_id $name }} {{- end }} \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index 07b70fb2..f22dcaef 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -8,6 +8,7 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} spec: + concurrencyPolicy: Forbid schedule: {{ .Values.cronSchedule }} jobTemplate: spec: @@ -69,7 +70,7 @@ spec: - | # Wait for the Spark pod to be ready SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') - kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master=local[*] --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" {{- end }} {{- with .Values.sidecars }} {{- toYaml . | nindent 12 }} diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml index 1c4c2fce..ceca090a 100644 --- a/command-service/helm-charts/spark-connector-cron/values.yaml +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -125,7 +125,7 @@ python_path: /opt/bitnami/python/bin/python spark: host: http://spark-master-svc.spark.svc.cluster.local:8998/batches/ master: - host: spark-master-svc.spark.svc.cluster.local + host: spark://spark-master-0.spark-headless.spark.svc.cluster.local:7077 port: 7077 driver: extraJavaOptions: -Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties diff --git a/command-service/requirements.txt b/command-service/requirements.txt index 6136ac6d..8593fc9f 100644 --- a/command-service/requirements.txt +++ b/command-service/requirements.txt @@ -16,4 +16,5 @@ tenacity==8.2.2 kafka-python-ng==2.2.2 boto3 prometheus-client +confluent-kafka==2.5.3 requests \ No newline at end of file diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py index 1c989a5c..3b6a82b9 100644 --- a/command-service/src/command/alert_manager_command.py +++ b/command-service/src/command/alert_manager_command.py @@ -142,7 +142,10 @@ def create_alert_metric( ) else: error_data = json.loads(response.body) - error_message = error_data["params"]["errmsg"] + if "params" in error_data and "errmsg" in error_data["params"]: + error_message = error_data["params"]["errmsg"] + else: + error_message = "Failed to publish alerts" return ActionResponse( status="ERROR", status_code=500, @@ -176,7 +179,7 @@ def create_alert_rule(self, payload: dict) -> ActionResponse: "subComponent": dataset_name, "metric": prom_metric, "operator": operator, - "threshold": threshold, + "threshold": [+threshold], "metricAlias": metric_alias, } }, @@ -195,7 +198,10 @@ def create_alert_rule(self, payload: dict) -> ActionResponse: self.publish_alert_rule(alert_id=alert_id) else: error_data = json.loads(response.body) - error_message = error_data["params"]["errmsg"] + if "params" in error_data and "errmsg" in error_data["params"]: + error_message = error_data["params"]["errmsg"] + else: + error_message = "Failed to publish alerts" return ActionResponse( status="ERROR", status_code=500, diff --git a/command-service/src/command/command_executor.py b/command-service/src/command/command_executor.py index 665e4636..dffa8155 100644 --- a/command-service/src/command/command_executor.py +++ b/command-service/src/command/command_executor.py @@ -10,6 +10,7 @@ from command.druid_command import DruidCommand from command.flink_command import FlinkCommand from command.telemetry_command import TelemetryCommand +from command.kafka_command import KafkaCommand from config import Config from model.data_models import Action, ActionResponse, CommandPayload from service.db_service import DatabaseService @@ -49,6 +50,9 @@ def __init__(self): self.db_command = DBCommand( db_service=self.db_service, dataset_command=self.dataset_command ) + self.create_kafka_topic = KafkaCommand( + config=self.config_obj, http_service=self.http_service, dataset_command=self.dataset_command + ) self.audit_event_command = TelemetryCommand( telemetry_service=self.telemetry_service, dataset_command=self.dataset_command, @@ -56,6 +60,7 @@ def __init__(self): self.action_commands = {} self.action_commands[Action.START_PIPELINE_JOBS.name] = self.flink_command self.action_commands[Action.MAKE_DATASET_LIVE.name] = self.db_command + self.action_commands[Action.CREATE_KAFKA_TOPIC.name] = self.create_kafka_topic self.action_commands[Action.SUBMIT_INGESTION_TASKS.name] = self.druid_command self.action_commands[Action.DEPLOY_CONNECTORS.name] = self.connector_command self.action_commands[Action.CREATE_ALERT_METRIC.name] = ( diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 7b2d2b13..8f5b145e 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -37,16 +37,16 @@ def execute(self, command_payload: CommandPayload, action: Action): return result def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): - result = None - self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"]) + result = None + self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"], active_connectors, dataset_id) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) - return result + return result - def _stop_connector_jobs(self, is_masterdata, namespace): + def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, dataset_id): print(f"Uninstalling jobs for {namespace}..") base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] - + # managed_releases = [] # connector_jar_config = self.config.find("connector_job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -61,13 +61,13 @@ def _stop_connector_jobs(self, is_masterdata, namespace): helm_ls_result = subprocess.run( helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if helm_ls_result.returncode == 0: - jobs = helm_ls_result.stdout.decode() - for job in jobs.splitlines()[1:]: - release_name = job.split()[0] - if base_helm_chart in job: - print("Uninstalling job {0}".format(release_name)) + jobs = helm_ls_result.stdout.decode().splitlines()[1:] + job_names = {job.split()[0] for job in jobs if base_helm_chart in job} + spark_connector = {connector.id for connector in active_connectors if connector.connector_runtime == "spark"} + for release_name in spark_connector: + if release_name in job_names: + print(f"Uninstalling job {release_name} related to dataset'{dataset_id}'...") helm_uninstall_cmd = [ "helm", "uninstall", @@ -81,12 +81,10 @@ def _stop_connector_jobs(self, is_masterdata, namespace): stderr=subprocess.PIPE, ) if helm_uninstall_result.returncode == 0: - print(f"Successfully uninstalled job {release_name}...") + print(f"Successfully uninstalled job '{release_name}'...") else: - print( - f"Error uninstalling job {release_name}: {helm_uninstall_result.stderr.decode()}" - ) - + print(f"Error uninstalling job '{release_name}': {helm_uninstall_result.stderr.decode()}") + def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None for connector in active_connectors: @@ -101,14 +99,14 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): f"Connector {connector.connector_id} is not supported for deployment" ) break - + # if is_masterdata: # print("Installing masterdata job") # masterdata_jar_config = self.config.find("masterdata_job") # for release in masterdata_jar_config: # result = self._perform_install(release) return result - + def _perform_flink_install(self, dataset_id, connector_instance): err = None result = None @@ -117,14 +115,14 @@ def _perform_flink_install(self, dataset_id, connector_instance): namespace = self.connector_job_config["flink"]["namespace"] job_name = release_name.replace(".", "-") helm_ls_cmd = ["helm", "ls", "--namespace", namespace] - + helm_ls_result = subprocess.run( helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - + if helm_ls_result.returncode == 0: jobs = helm_ls_result.stdout.decode() - + deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) if deployment_exists: restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( @@ -148,14 +146,14 @@ def _perform_flink_install(self, dataset_id, connector_instance): error_message="FLINK_HELM_LIST_EXCEPTION", ) print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") - + if err is None: result = ActionResponse(status="OK", status_code=200) return result else: if self._get_live_instances(runtime="flink", connector_instance=connector_instance): - connector_source = json.loads(connector_instance.connector_source) + connector_source = connector_instance.connector_source flink_jobs = dict() flink_jobs[job_name] = { "enabled": "true", @@ -163,7 +161,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): "source": connector_source.get("source"), "main_program": connector_source.get("main_program") } - + set_json_value = json.dumps(flink_jobs) helm_install_cmd = [ "helm", @@ -175,17 +173,17 @@ def _perform_flink_install(self, dataset_id, connector_instance): namespace, "--create-namespace", "--set-json", - f"flink_jobs={set_json_value.replace(" ", "")}" + f"""flink_jobs={set_json_value.replace(" ", "")}""" ] - + print("flink connector installation: ", " ".join(helm_install_cmd)) helm_install_result = subprocess.run( helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - + print(helm_install_result) - + if helm_install_result.returncode == 0: print(f"Job '{job_name}' deployment succeeded...") else: @@ -212,14 +210,14 @@ def _perform_flink_install(self, dataset_id, connector_instance): status_code=500, error_message="FLINK_HELM_LIST_EXCEPTION", ) - + def _perform_spark_install(self, dataset_id, connector_instance): err = None result = None release_name = connector_instance.id - connector_source = json.loads(connector_instance.connector_source) + connector_source = connector_instance.connector_source schedule = connector_instance.operations_config["schedule"] - + schedule_configs = { "Hourly": "0 * * * *", # Runs at the start of every hour "Weekly": "0 0 * * 0", # Runs at midnight every Sunday @@ -228,7 +226,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): } namespace = self.connector_job_config["spark"]["namespace"] - + helm_install_cmd = [ "helm", "upgrade", @@ -268,7 +266,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", ) print(f"Error updating job '{release_name}': {helm_install_result.stderr.decode()}") - + if result is None: result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") @@ -277,7 +275,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): def _get_connector_details(self, dataset_id): active_connectors = [] query = f""" - SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology, cr.version + SELECT ci.id, ci.connector_id, ci.dataset_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology, cr.version FROM connector_instances ci JOIN connector_registry cr on ci.connector_id = cr.id WHERE ci.status= %s and ci.dataset_id = %s @@ -305,11 +303,11 @@ def _get_masterdata_details(self, dataset_id): is_masterdata = True return is_masterdata - + ## TODO: check for connector_id as well def _get_live_instances(self, runtime, connector_instance): has_live_instances = False - query = f""" + query = f""" SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id FROM connector_instances ci JOIN connector_registry cr ON ci.connector_id = cr.id @@ -320,9 +318,9 @@ def _get_live_instances(self, runtime, connector_instance): rows = self.db_service.execute_select_all(sql=query, params=params) if len(rows) > 0: has_live_instances = True - + return has_live_instances - + # def _perform_install(self, release): # err = None # result = None diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index a91f651d..40edf5f7 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -8,6 +8,7 @@ from pathlib import Path import requests +import subprocess from fastapi import status from config import Config @@ -176,6 +177,8 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: result = [] tenant = self.metadata.get("metadata", {}).get("tenant", "") + self.copy_connector_to_runtime(self.metadata['metadata']['runtime'], connector_source) + if tenant == "multiple": connector_objects = self.metadata["connectors"] for obj in connector_objects: @@ -210,13 +213,17 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: ) query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + if not success: return RegistryResponse( status="failure", message=f"Failed to register connector {connector_id}", statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - result.append(registry_meta.to_dict()) + result.append(registry_meta.to_dict()) return RegistryResponse( status="success", connector_info=result, @@ -259,6 +266,10 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: ) query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + if not success: return RegistryResponse( status="failure", @@ -272,9 +283,11 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: statusCode=status.HTTP_200_OK, ) + def execute_query(self, query, params) -> bool: try: result = self.db_service.execute_upsert(sql=query, params=params) + print(f"Connector Registry | {result} rows affected") return result > 0 # Assuming the result is the number of affected rows except Exception as e: print( @@ -312,16 +325,16 @@ def build_insert_query(self, registry_meta: ConnectorRegsitryv2): ui_spec_json = json.dumps(registry_meta.ui_spec) query =f""" INSERT INTO connector_registry ( - id, connector_id, name, type, category, version, description, - technology, runtime, licence, owner, iconurl, status, source_url, + id, connector_id, name, type, category, version, description, + technology, runtime, licence, owner, iconurl, status, source_url, source, ui_spec, created_by, updated_by, created_date, updated_date, live_date ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, - %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ON CONFLICT ( connector_id, version - ) DO UPDATE SET + ) DO UPDATE SET id = %s, name = %s, type = %s, @@ -383,14 +396,26 @@ def build_insert_query(self, registry_meta: ConnectorRegsitryv2): datetime.now(), ) return query, params - + def load_file_bytes(self, rel_path: str) -> bytes | None: file_path = Path(self.extraction_path) for item in file_path.glob("*/{}".format(rel_path)): try: + prefixes = { + ".svg": "data:image/svg+xml;base64,", + ".jpeg": "data:image/jpeg;base64,", + ".jpg": "data:image/jpg;base64,", + ".gif": "data:image/gif;base64,", + ".webp": "data:image/webp;base64,", + ".ico": "data:image/x-icon;base64," + } + + prefix = prefixes.get(item.suffix, "data:application/octet-stream;base64,") + print(f"Connector Registry | Image Suffix: {item.suffix} Base64 Prefix in Use: {prefix}") + with open(item, 'rb') as file: file_content = file.read() - encoded = base64.b64encode(file_content).decode("ascii") + encoded = (prefix + base64.b64encode(file_content).decode("ascii")).strip() except IsADirectoryError: print( f"Connector Registry | No value for icon URL given at metadata: {rel_path}" @@ -414,13 +439,93 @@ def update_connector_registry(self, _id, ver): f"UPDATE connector_registry SET status = 'Retired', updated_date = now() WHERE connector_id = %s AND status = 'Live' AND version != %s", (_id, ver) ) print( - f"Connector Registry | Updated {result} existing rows with connector_id: {_id} and version: {ver}" + f"Connector Registry | Retired {result} versions for connector_id: {_id} and version: {ver}" ) except Exception as e: print( f"Connector Registry | An error occurred during the execution of Query: {e}" ) + def copy_connector_to_runtime(self, runtime: str, connector_source: str): + if runtime == "spark": + return self.copy_connector_to_spark(connector_source) + + + def copy_connector_to_spark(self, connector_source: str): + print(f"Connector Registry | copying {connector_source} to spark") + ## get name of the spark pod using kubectl + spark_pod_cmd = [ + "kubectl", + "get", + "pods", + "-n", + "spark", + "-l", + "app.kubernetes.io/name=spark,app.kubernetes.io/component=master", + "-o", + "jsonpath='{.items[0].metadata.name}'", + ] + + spark_pod_result = subprocess.run(spark_pod_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + print(f"Connector Registry | spark_pod_result: {spark_pod_result}") + + if spark_pod_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to get the spark pod", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + spark_pod = spark_pod_result.stdout.decode("utf-8").replace("'", "") + print(f"Connector Registry | spark_pod: {spark_pod}") + + ## copy the connector to the spark pod under /data/connectors/{source} + source_path = os.path.join(self.extraction_path, connector_source) + copy_cmd = [ + "kubectl", + "cp", + f"{source_path}", + f"spark/{spark_pod}:/data/connectors/{connector_source}", + ] + + copy_result = subprocess.run(copy_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Connector Registry | copy_result: {copy_result}") + if copy_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to copy the connector to spark", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + if self.metadata['metadata']['technology'] == "python": + pip_install_cmd = [ + "kubectl", + "exec", + f"pod/{spark_pod}", + "-n", + "spark", + "--", + "bash", + "-c", + f"pip install -r /data/connectors/{connector_source}/requirements.txt", + ] + + pip_install_result = subprocess.run(pip_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Connector Registry | pip_install_result: {pip_install_result}") + if pip_install_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to install the requirements on spark", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + return RegistryResponse( + status="success", + message="connector copied to spark successfully", + statusCode=status.HTTP_200_OK, + ) + class ExtractionUtil: def extract_gz(tar_path, extract_path): with tarfile.open(tar_path, "r:*") as tar: diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index db7afa7d..d0433884 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -26,7 +26,7 @@ def __init__( self.http_service = http_service self.config_service_host = self.config.find("config_service.host") self.config_service_port = self.config.find("config_service.port") - self.base_url = f"http://{self.config_service_host}:{self.config_service_port}/datasets/v1/export" + self.base_url = f"http://{self.config_service_host}:{self.config_service_port}/v2/datasets/export" def _get_draft_dataset_record(self, dataset_id): query = f""" @@ -66,10 +66,9 @@ def _check_for_live_record(self, dataset_id): def audit_live_dataset(self, command_payload: CommandPayload, ts: int): dataset_id = command_payload.dataset_id dataset_record, data_version = self._check_for_live_record(dataset_id) - export_dataset = self.http_service.post( - url=self.base_url, - body=json.dumps({"dataset_id": dataset_id}), - headers={"Content-Type": "application/json"}, + url=self.base_url + '/{}'.format(dataset_id) + export_dataset = self.http_service.get( + url=url ) if export_dataset.status == 200: result = json.loads(export_dataset.body) @@ -103,5 +102,4 @@ def audit_live_dataset(self, command_payload: CommandPayload, ts: int): "Failed to get dataset configurations from export API, dataset_id: ", dataset_id, ) - print("Dataset record details: ", json.dumps(dataset_record)) return False diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 4cb62e2c..6279102f 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -48,7 +48,6 @@ def _change_dataset_to_active(self, command_payload: CommandPayload): self._insert_datasource_record(dataset_id, draft_dataset_id) self._insert_connector_instances(dataset_id, draft_dataset_record) self._insert_dataset_transformations(dataset_id, draft_dataset_record) - self._delete_draft_dataset(dataset_id, draft_dataset_id) return ActionResponse(status="OK", status_code=200) else: return ActionResponse( @@ -176,7 +175,7 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): draft_datasource.datasource, dataset_id, draft_datasource.datasource_ref, - json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + json.dumps(draft_datasource.ingestion_spec), draft_datasource.type, json.dumps(draft_datasource.retention_period).replace("'", "''"), json.dumps(draft_datasource.archival_policy).replace("'", "''"), @@ -191,7 +190,7 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): json.dumps(draft_datasource.metadata).replace("'", "''"), draft_datasource.datasource, - json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + json.dumps(draft_datasource.ingestion_spec), draft_datasource.type, json.dumps(draft_datasource.retention_period).replace("'", "''"), json.dumps(draft_datasource.archival_policy).replace("'", "''"), @@ -258,14 +257,14 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): data_class = DatasetConnectorConfigDraft, data = record ) current_timestamp = dt.now() + operations_config = connector_config.operations_config if connector_config.operations_config is not None else {} if connector_config.version == 'v2': params = ( connector_config.id, dataset_id, connector_config.connector_id, - json.dumps(connector_config.connector_config).replace("'", "''"), - json.dumps(connector_config.operations_config).replace("'", "''"), - connector_config.data_format, + connector_config.connector_config, + json.dumps(operations_config).replace("'", "''"), DatasetStatusType.Live.name, json.dumps(emptyJson), json.dumps(emptyJson), @@ -274,10 +273,8 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): current_timestamp, current_timestamp, current_timestamp, - - json.dumps(connector_config.connector_config).replace("'", "''"), + connector_config.connector_config, json.dumps(connector_config.operations_config).replace("'", "''"), - connector_config.data_format, draft_dataset_record.get('updated_by'), current_timestamp, current_timestamp, @@ -285,7 +282,7 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): ) insert_query = f""" INSERT INTO connector_instances(id, dataset_id, connector_id, connector_config, operations_config, - data_format, status, connector_state, connector_stats, created_by, updated_by, created_date, + status, connector_state, connector_stats, created_by, updated_by, created_date, updated_date, published_date) VALUES ( %s, @@ -300,13 +297,11 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): %s, %s, %s, - %s, %s ) ON CONFLICT (id) DO UPDATE SET connector_config = %s, operations_config = %s, - data_format = %s, updated_by = %s, updated_date = %s, published_date = %s, @@ -412,18 +407,4 @@ def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): """ result = self.db_service.execute_upsert(sql=insert_query, params=params) print(f"Dataset Transformation {dataset_id + '_' + transformation.field_key} record inserted successfully...") - return result - - def _delete_draft_dataset(self, dataset_id, draft_dataset_id): - - self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) - print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") - - self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations_draft where dataset_id = %s""", params=(draft_dataset_id,)) - print(f"Draft transformations/tables for {dataset_id} are deleted successfully...") - - self.db_service.execute_delete(sql=f"""DELETE from dataset_source_config_draft where dataset_id = %s""", params=(draft_dataset_id,)) - print(f"Draft source config/tables for {dataset_id} are deleted successfully...") - - self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) - print(f"Draft Dataset for {dataset_id} is deleted successfully...") \ No newline at end of file + return result \ No newline at end of file diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py index 901b18ad..73d5bf86 100644 --- a/command-service/src/command/druid_command.py +++ b/command-service/src/command/druid_command.py @@ -34,15 +34,23 @@ def _submit_ingestion_task(self, dataset_id): print( f"Invoking SUBMIT_INGESTION_TASKS command for dataset_id {dataset_id}..." ) + task_submitted = 1 for record in datasources_records: - if record["dataset_type"] == "dataset" and record["type"] == "druid": + if record["dataset_type"] == "event" and record["type"] == "druid": print(f"Submitting ingestion task for datasource ...") ingestion_spec = json.dumps(record["ingestion_spec"]) response = self.http_service.post( url=f"{self.router_url}/{self.supervisor_endpoint}", body=ingestion_spec, - headers={"Content-Type": "application/json"}, + headers={"Content-Type": "application/json"} ) + if response.status != 200: + task_submitted = 0 + break + if task_submitted: + query=f"SELECT id FROM datasets_draft WHERE dataset_id= %s" + response = self.db_service.execute_select_one(sql=query, params=(dataset_id,)) + self._delete_draft_dataset(dataset_id, response[0]) return ActionResponse(status="OK", status_code=200) else: print( @@ -51,3 +59,17 @@ def _submit_ingestion_task(self, dataset_id): return ActionResponse( status="ERROR", status_code=404, error_message="DATASET_ID_NOT_FOUND" ) + + def _delete_draft_dataset(self, dataset_id, draft_dataset_id): + + self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft transformations/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_source_config_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft source config/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) + print(f"Draft Dataset for {dataset_id} is deleted successfully...") diff --git a/command-service/src/command/kafka_command.py b/command-service/src/command/kafka_command.py new file mode 100644 index 00000000..60578d5c --- /dev/null +++ b/command-service/src/command/kafka_command.py @@ -0,0 +1,54 @@ +from command.icommand import ICommand +from command.icommand import ICommand +from config import Config +from model.data_models import Action, ActionResponse, CommandPayload +from service.http_service import HttpService +from confluent_kafka.admin import AdminClient, NewTopic, KafkaError, KafkaException +from command.dataset_command import DatasetCommand + +class KafkaCommand(ICommand): + def __init__(self, config: Config, http_service: HttpService, dataset_command: DatasetCommand): + self.config = config + self.http_service = http_service + self.dataset_command = dataset_command + + def execute(self, command_payload: CommandPayload, action: Action): + result = None + dataset_id = command_payload.dataset_id + live_dataset, data_version = self.dataset_command._check_for_live_record(dataset_id) + if live_dataset is None: + if action == Action.CREATE_KAFKA_TOPIC.name: + print( + f"Invoking CREATE_KAFKA_TOPIC command for dataset_id {dataset_id}..." + ) + draft_dataset_record = self.dataset_command._get_draft_dataset(dataset_id) + topic = draft_dataset_record.get("router_config")["topic"] + broker = self.config.find("kafka.brokers") + num_partitions = self.config.find("kafka.no_of_partitions") + replication_factor = self.config.find("kafka.replication_factor") + print(f"topic is", topic) + result = self.create_kafka_topic(topic, broker, num_partitions, replication_factor) + return result + return ActionResponse(status="OK", status_code=200) + + + def create_kafka_topic(self, topic, broker, num_partitions, replication_factor): + errValue = ActionResponse(status="ERROR", status_code=500) + + try: + admin_client = AdminClient({'bootstrap.servers': broker}) + new_topic = [NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor)] + fs = admin_client.create_topics(new_topic) + for topic, f in fs.items(): + f.result() + print(f"Topic '{topic}' created successfully") + return ActionResponse(status="OK", status_code=200) + except (KafkaError, KafkaException) as kafkaErr: + print(f"Kafka exception:", kafkaErr) + return errValue + except RuntimeError as e: + print(f"Runtime exception: {e}") + return errValue + except Exception as e: + print(f"Error creating topic: {e}") + return errValue \ No newline at end of file diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index 2c432ba0..80276238 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -15,6 +15,7 @@ flink: commands: PUBLISH_DATASET: workflow: + - CREATE_KAFKA_TOPIC - MAKE_DATASET_LIVE - SUBMIT_INGESTION_TASKS - START_PIPELINE_JOBS @@ -117,7 +118,7 @@ config_service: port: 4000 druid: - router_host: localhost + router_host: http://localhost router_port: 8888 supervisor_endpoint: indexer/v1/supervisor @@ -166,6 +167,8 @@ kafka: brokers: localhost:9092 telemetry: topic: system.telemetry.events + replication_factor: 1 + no_of_partitions: 1 connector_jobs: spark: diff --git a/command-service/src/model/data_models.py b/command-service/src/model/data_models.py index f248e6ba..2ab1611d 100644 --- a/command-service/src/model/data_models.py +++ b/command-service/src/model/data_models.py @@ -14,6 +14,7 @@ class Command(Enum): class Action(Enum): SUBMIT_INGESTION_TASKS = "SUBMIT_INGESTION_TASKS" MAKE_DATASET_LIVE = "MAKE_DATASET_LIVE" + CREATE_KAFKA_TOPIC = "CREATE_KAFKA_TOPIC" MAKE_DATASOURCE_ACTIVE = "MAKE_DATASOURCE_ACTIVE" START_PIPELINE_JOBS = "START_PIPELINE_JOBS" CREATE_ALERT_METRIC = "CREATE_ALERT_METRIC" diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index f77f8314..65abb534 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -79,7 +79,7 @@ class DatasourcesDraft: class DatasetConnectorConfigDraft: id: str connector_id: str - connector_config: str | None + connector_config: str | dict version: str operations_config: dict | None = None data_format: str | None = 'json' @@ -121,7 +121,8 @@ class ConnectorRegsitryv2: class ConnectorInstance: id: str connector_id: str + dataset_id: str operations_config: dict connector_runtime: str - connector_source: str + connector_source: dict technology: str \ No newline at end of file diff --git a/command-service/src/service/db_service.py b/command-service/src/service/db_service.py index ded56b91..275f11c6 100644 --- a/command-service/src/service/db_service.py +++ b/command-service/src/service/db_service.py @@ -6,7 +6,7 @@ from config import Config def reconnect(func: Callable): - + def wrapper(db_connection, *args, **kwargs): tdecorator = retry(wait=wait_exponential(), stop=stop_after_attempt(3)) decorated = tdecorator(func) @@ -60,9 +60,9 @@ def execute_upsert(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) cursor.execute(sql, params) + db_connection.commit() record_count = cursor.rowcount db_connection.close() - # print(f"{record_count} inserted/updated successfully") return record_count # @reconnect