From 5a3d0d29079ba82a468cf1076967907d397072c4 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Tue, 19 Mar 2024 12:27:28 +0530 Subject: [PATCH 1/2] Release 1.0.4-GA (#105) * default config for dataset_config (#99) * Sanketika-Obsrv/issue-tracker#14: feat: validate sql query (#100) --------- Co-authored-by: harishkumar gangula Co-authored-by: Shreyas Bhaktharam <121869503+shreyasb22@users.noreply.github.com> --- .../resources/schemas/DatasetConfigDefault.ts | 20 ++++++++ api-service/src/test/Fixtures.ts | 4 +- api-service/src/test/QueryTestService.spec.ts | 50 +++++++++++++++++++ api-service/src/validators/QueryValidator.ts | 37 ++++++++++---- 4 files changed, 100 insertions(+), 11 deletions(-) diff --git a/api-service/src/resources/schemas/DatasetConfigDefault.ts b/api-service/src/resources/schemas/DatasetConfigDefault.ts index 1741f750..f5ba0576 100644 --- a/api-service/src/resources/schemas/DatasetConfigDefault.ts +++ b/api-service/src/resources/schemas/DatasetConfigDefault.ts @@ -3,6 +3,16 @@ import { DatasetStatus, ValidationMode } from "../../models/DatasetModels"; export const defaultConfig = { "master": { + "dataset_config": { + "data_key": "", + "timestamp_key": "", + "exclude_fields": [], + "entry_topic": config.telemetry_service_config.kafka.topics.createMasterDataset, + "redis_db_host": config.redis_config.redis_host, + "redis_db_port": config.redis_config.redis_port, + "index_data": true, + "redis_db": 3 + }, "validation_config": { "validate": true, "mode": ValidationMode.Strict, @@ -52,6 +62,16 @@ export const defaultConfig = { "topic": "" }, "tags": [], + "dataset_config": { + "data_key": "", + "timestamp_key": "", + "exclude_fields": [], + "entry_topic": config.telemetry_service_config.kafka.topics.createDataset, + "redis_db_host": config.redis_config.redis_host, + "redis_db_port": config.redis_config.redis_port, + "index_data": true, + "redis_db": 0 // The default Redis database index. + }, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM" diff --git a/api-service/src/test/Fixtures.ts b/api-service/src/test/Fixtures.ts index a69fd39c..c391073b 100644 --- a/api-service/src/test/Fixtures.ts +++ b/api-service/src/test/Fixtures.ts @@ -35,6 +35,8 @@ class TestDruidQuery { '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT __time FROM \\"invalid-datasource\\" LIMIT 10"}}'; public static SKIP_VALIDATION_NATIVE = '{"context":{"dataSource":"system-stats"},"query":{"queryType":"timeBoundary","dataSource":"system-stats","granularity":"all","intervals":["2022-10-17/2022-10-19"],"resultFormat":"compactedList","columns":["__time","scans"],"metrics":{"type":"numeric","metric":"count"},"aggregations":[{"type":"count","name":"count"}]}}'; public static SKIP_VALIDATION_SQL = '{"context":{"dataSource":"failed-events-summary"},"querySql":{"query":"SELECT * FROM \\"failed-events-summary\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}}'; + public static INVALID_SQL_QUERY = '{\"context\":{\"dataSource\":\"system-events\",\"granularity\":\"day\"},\"querySql\":{\"query\":\"SELECT * \"}}'; + public static MISSING_TABLE_NAME = '{\"context\":{\"dataSource\":\"system-events\",\"granularity\":\"day\"},\"querySql\":{\"query\":\"SELECT * FROM \"}}'; } class TestDataIngestion { @@ -142,4 +144,4 @@ class TestExhaust { } } -export { TestDruidQuery, TestDataIngestion, TestDataset, TestDataSource, TestDatasetSourceConfig, TestExhaust, TestSubmitIngestion}; +export { TestDruidQuery, TestDataIngestion, TestDataset, TestDataSource, TestDatasetSourceConfig, TestExhaust, TestSubmitIngestion}; \ No newline at end of file diff --git a/api-service/src/test/QueryTestService.spec.ts b/api-service/src/test/QueryTestService.spec.ts index 4e24c538..a65a829b 100644 --- a/api-service/src/test/QueryTestService.spec.ts +++ b/api-service/src/test/QueryTestService.spec.ts @@ -524,6 +524,56 @@ describe("QUERY API", () => { done(); }) }) + it("should throw error for invalid sql query", (done) => { + chai.spy.on(dbConnector, "readRecords", () => { + return [{ "datasource_ref": "sample_ref" }] + }) + nock(config.druidHost + ":" + config.druidPort) + .get(config.druidDatasourcesEndPoint) + .reply(200, ["sample_ref"]) + nock(config.druidHost + ":" + config.druidPort) + .post(config.druidSqlEndPoint) + .reply(200, [{ events: [] }]); + chai. + request(app) + .post(config.apiDruidSqlEndPoint) + .send(JSON.parse(TestDruidQuery.INVALID_SQL_QUERY)) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object"); + res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); + res.body.params.status.should.be.eq(constants.STATUS.FAILURE); + res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); + chai.spy.restore(dbConnector, "readRecords"); + nock.cleanAll(); + done(); + }) + }) + it("should throw error is table name is missing from the SQL Query", (done) => { + chai.spy.on(dbConnector, "readRecords", () => { + return [{ "datasource_ref": "sample_ref" }] + }) + nock(config.druidHost + ":" + config.druidPort) + .get(config.druidDatasourcesEndPoint) + .reply(200, ["sample_ref"]) + nock(config.druidHost + ":" + config.druidPort) + .post(config.druidSqlEndPoint) + .reply(200, [{ events: [] }]); + chai. + request(app) + .post(config.apiDruidSqlEndPoint) + .send(JSON.parse(TestDruidQuery.MISSING_TABLE_NAME)) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object"); + res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); + res.body.params.status.should.be.eq(constants.STATUS.FAILURE); + res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); + chai.spy.restore(dbConnector, "readRecords"); + nock.cleanAll(); + done(); + }) + }) }) describe("error scenarios", () => { it("should handle the error", (done) => { diff --git a/api-service/src/validators/QueryValidator.ts b/api-service/src/validators/QueryValidator.ts index d207f446..bf306e0b 100644 --- a/api-service/src/validators/QueryValidator.ts +++ b/api-service/src/validators/QueryValidator.ts @@ -1,6 +1,6 @@ import httpStatus from "http-status"; -import _ from "lodash"; -import moment, { Moment} from "moment"; +import _ from "lodash"; +import moment, { Moment } from "moment"; import { queryRules } from "../configs/QueryRules"; import { IConnector, IValidator } from "../models/DatasetModels"; import { ICommonRules, ILimits, IQuery, IQueryTypeRules, IRules } from "../models/QueryModels"; @@ -30,9 +30,12 @@ export class QueryValidator implements IValidator { return validationStatus.isValid ? (shouldSkip ? validationStatus : this.setDatasourceRef(dataSource, data)) : validationStatus case routesConfig.query.sql_query.api_id: validationStatus = await this.validateSqlQuery(data) - dataSource = this.getDataSource(data) - shouldSkip = _.includes(config.exclude_datasource_validation, dataSource); - return validationStatus.isValid ? (shouldSkip ? validationStatus : this.setDatasourceRef(dataSource, data)) : validationStatus + if (validationStatus.isValid) { + dataSource = this.getDataSource(data) + shouldSkip = _.includes(config.exclude_datasource_validation, dataSource); + return validationStatus.isValid ? (shouldSkip ? validationStatus : this.setDatasourceRef(dataSource, data)) : validationStatus + } + return validationStatus default: return { isValid: false } } @@ -45,17 +48,31 @@ export class QueryValidator implements IValidator { try { return (!_.isEmpty(dataSourceLimits)) ? this.validateQueryRules(queryObj, dataSourceLimits.queryRules[queryObj.query.queryType as keyof IQueryTypeRules]) : { isValid: true } } catch (error: any) { - return { isValid: false, message: error.message || "error ocuured while validating native query", code: error.code || httpStatus[ "400_NAME" ] }; + return { isValid: false, message: error.message || "error ocuured while validating native query", code: error.code || httpStatus["400_NAME"] }; } } - private validateSqlQuery(data: any): ValidationStatus { - this.setQueryLimits(data, this.limits.common); - let dataSourceLimits = this.getDataSourceLimits(this.getDataSource(data)); + private validateSqlQuery(data: IQuery): ValidationStatus { try { + let query = data.querySql.query; + if (_.isEmpty(query)) { + return { isValid: false, message: "Query must not be empty", code: httpStatus["400_NAME"] }; + } + const fromClause = /\bFROM\b/i; + const isFromClausePresent = fromClause.test(query) + if (!isFromClausePresent) { + return { isValid: false, message: "Invalid SQL Query", code: httpStatus["400_NAME"] }; + } + const dataset = query.substring(query.indexOf("FROM")).split(" ")[1].replace(/\\/g, ""); + if (_.isEmpty(dataset)) { + return { isValid: false, message: "Dataset name must be present in the SQL Query", code: httpStatus["400_NAME"] }; + } + this.setQueryLimits(data, this.limits.common); + let datasource = this.getDataSource(data); + let dataSourceLimits = this.getDataSourceLimits(datasource); return (!_.isEmpty(dataSourceLimits)) ? this.validateQueryRules(data, dataSourceLimits.queryRules.scan) : { isValid: true }; } catch (error: any) { - return { isValid: false, message: error.message || "error ocuured while validating native query", code: error.code || httpStatus[ "400_NAME" ] }; + return { isValid: false, message: error.message || "error ocuured while validating SQL query", code: error.code || httpStatus[ "500_NAME" ] }; } } From 9d5fbd8b5834458a7a1a6d2abccd1dd840dda585 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Tue, 23 Apr 2024 17:42:45 +0530 Subject: [PATCH 2/2] Release 1.0.5-GA (#147) --- .github/workflows/codeql.yaml | 84 +++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 .github/workflows/codeql.yaml diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 00000000..e8cc9f73 --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,84 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "develop", "main" ] + pull_request: + branches: [ "develop", "main" ] + + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + language: [ 'javascript-typescript' ] + # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] + # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" +