From 76f0344b3feeeb282530ffdb84545bbcdb1b8050 Mon Sep 17 00:00:00 2001 From: enudler Date: Tue, 14 Jan 2020 18:07:55 +0200 Subject: [PATCH] feat(processors): adding exported functions to processors (#255) * feat(processors): adding exported functions to processors --- docs/openapi3.yaml | 8 ++ package.json | 1 - .../17__processors_exported_functions.cql | 1 + .../03_tests_processor_exported_functions.js | 15 ++++ .../database/cassandra/cassandraConnector.js | 10 +-- .../database/sequelize/sequelizeConnector.js | 14 +++- src/processors/models/processorsManager.js | 57 +++++++++------ src/tests/models/fileManager.js | 18 +---- .../jobs/createJobDocker-test.js | 4 +- .../processors/processors-test.js | 73 ++++++++++++++++++- tests/integration-tests/runLocal.sh | 2 +- .../models/processorsManager-test.js | 17 ++++- .../tests/models/fileManager-test.js | 36 --------- 13 files changed, 167 insertions(+), 89 deletions(-) create mode 100644 src/database/cassandra-handler/init-scripts/17__processors_exported_functions.cql create mode 100644 src/database/sequlize-handler/migrations/03_tests_processor_exported_functions.js delete mode 100644 tests/unit-tests/tests/models/fileManager-test.js diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index 78676b7db..21d287cd7 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -1326,6 +1326,7 @@ components: properties: name: type: string + minLength: 1 description: The name of the test. example: Order from Pet Store processor_file_url: @@ -1758,12 +1759,19 @@ components: readOnly: true name: type: string + minLength: 1 description: The name of the processor. example: Custom javascript for logging description: type: string description: A description of the processor. example: logs every error (5xx). + exported_functions: + type: array + readOnly: true + description: Names of all exported function in the javascript file + items: + type: string updated_at: type: string format: date-time diff --git a/package.json b/package.json index 9d4822844..c8aaa3e27 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,6 @@ "copy-dir": "^0.3.0", "cron": "^1.7.1", "dockerode": "^2.5.8", - "esprima": "^4.0.1", "express": "^4.17.1", "express-ajv-swagger-validation": "^0.9.0", "express-easy-zip": "^1.1.4", diff --git a/src/database/cassandra-handler/init-scripts/17__processors_exported_functions.cql b/src/database/cassandra-handler/init-scripts/17__processors_exported_functions.cql new file mode 100644 index 000000000..1eac2dda1 --- /dev/null +++ b/src/database/cassandra-handler/init-scripts/17__processors_exported_functions.cql @@ -0,0 +1 @@ +ALTER TABLE processors ADD exported_functions list; diff --git a/src/database/sequlize-handler/migrations/03_tests_processor_exported_functions.js b/src/database/sequlize-handler/migrations/03_tests_processor_exported_functions.js new file mode 100644 index 000000000..06fde6055 --- /dev/null +++ b/src/database/sequlize-handler/migrations/03_tests_processor_exported_functions.js @@ -0,0 +1,15 @@ +const Sequelize = require('sequelize'); + +module.exports.up = async (query, DataTypes) => { + let testsTable = await query.describeTable('processors'); + + if (!testsTable.exported_functions) { + await query.addColumn( + 'processors', 'exported_functions', + Sequelize.DataTypes.STRING); + } +}; + +module.exports.down = async (query, DataTypes) => { + await query.removeColumn('processors', 'exported_functions'); +}; diff --git a/src/processors/models/database/cassandra/cassandraConnector.js b/src/processors/models/database/cassandra/cassandraConnector.js index 72317ec13..4ab2399f9 100644 --- a/src/processors/models/database/cassandra/cassandraConnector.js +++ b/src/processors/models/database/cassandra/cassandraConnector.js @@ -3,11 +3,11 @@ let databaseConfig = require('../../../../config/databaseConfig'); let _ = require('lodash'); let client; -const INSERT_PROCESSOR = 'INSERT INTO processors(id, name, description, javascript, created_at, updated_at) values(?,?,?,?,?,?)'; +const INSERT_PROCESSOR = 'INSERT INTO processors(id, name, description, javascript, exported_functions, created_at, updated_at) values(?,?,?,?,?,?,?)'; const GET_ALL_PROCESSORS = 'SELECT * FROM processors'; const GET_PROCESSOR_BY_ID = 'SELECT * FROM processors WHERE id=?'; const DELETE_PROCESSOR = 'DELETE FROM processors WHERE id=?'; -const UPDATE_PROCESSOR = 'UPDATE processors SET name=?, description=?, javascript=?, updated_at=? WHERE id=? AND created_at=? IF EXISTS'; +const UPDATE_PROCESSOR = 'UPDATE processors SET name=?, description=?, javascript=?, exported_functions=?, updated_at=? WHERE id=? AND created_at=? IF EXISTS'; const INSERT_PROCESSOR_MAPPING = 'INSERT INTO processors_mapping(name, id) VALUES(?, ?)'; const DELETE_PROCESSOR_MAPPING = 'DELETE FROM processors_mapping WHERE name=?'; @@ -72,7 +72,7 @@ async function deleteProcessor(processorId) { } async function insertProcessor(processorId, processorInfo) { - let params = [processorId, processorInfo.name, processorInfo.description, processorInfo.javascript, Date.now(), Date.now()]; + let params = [processorId, processorInfo.name, processorInfo.description, processorInfo.javascript, processorInfo.exported_functions, Date.now(), Date.now()]; let mappingParams = [processorInfo.name, processorId]; const [processor] = await Promise.all([ executeQuery(INSERT_PROCESSOR, params, queryOptions), @@ -82,9 +82,9 @@ async function insertProcessor(processorId, processorInfo) { } async function updateProcessor(processorId, updatedProcessor) { - const { name, description, javascript, created_at: createdAt } = updatedProcessor; + const { name, description, javascript, exported_functions, created_at: createdAt } = updatedProcessor; const processor = await getProcessorById(processorId); - const params = [ name, description, javascript, Date.now(), processorId, createdAt.getTime() ]; + const params = [ name, description, javascript, exported_functions, Date.now(), processorId, createdAt.getTime() ]; return Promise.all([ executeQuery(UPDATE_PROCESSOR, params, queryOptions), executeQuery(INSERT_PROCESSOR_MAPPING, [updatedProcessor.name, processorId]), diff --git a/src/processors/models/database/sequelize/sequelizeConnector.js b/src/processors/models/database/sequelize/sequelizeConnector.js index 96280d477..1bffd1d72 100644 --- a/src/processors/models/database/sequelize/sequelizeConnector.js +++ b/src/processors/models/database/sequelize/sequelizeConnector.js @@ -25,6 +25,7 @@ async function insertProcessor(processorId, processorInfo) { name: processorInfo.name, description: processorInfo.description, javascript: processorInfo.javascript, + exported_functions: processorInfo.exported_functions, created_at: Date.now(), updated_at: Date.now() }; @@ -67,8 +68,8 @@ async function deleteProcessor(processorId) { async function updateProcessor(processorId, updatedProcessor) { const processorsModel = client.model('processor'); - const { name, description, javascript } = updatedProcessor; - return processorsModel.update({ name, description, javascript, updated_at: Date.now() }, { where: { id: processorId } }); + const { name, description, javascript, exported_functions } = updatedProcessor; + return processorsModel.update({ name, description, javascript, updated_at: Date.now(), exported_functions }, { where: { id: processorId } }); } async function initSchemas() { @@ -91,6 +92,15 @@ async function initSchemas() { }, updated_at: { type: Sequelize.DataTypes.DATE + }, + exported_functions: { + type: Sequelize.DataTypes.STRING, + get: function() { + return JSON.parse(this.getDataValue('exported_functions')); + }, + set: function(val) { + return this.setDataValue('exported_functions', JSON.stringify(val)); + } } }); await processorsFiles.sync(); diff --git a/src/processors/models/processorsManager.js b/src/processors/models/processorsManager.js index ac7b3e8e2..8a3fdf9a6 100644 --- a/src/processors/models/processorsManager.js +++ b/src/processors/models/processorsManager.js @@ -4,19 +4,18 @@ const uuid = require('uuid'); const logger = require('../../common/logger'), databaseConnector = require('./database/databaseConnector'), - fileManager = require('../../tests/models/fileManager.js'), + testsManager = require('../../tests/models/manager'), { ERROR_MESSAGES } = require('../../common/consts'); -let testsManager = require('../../tests/models/manager'); - module.exports.createProcessor = async function (processor) { const processorWithTheSameName = await databaseConnector.getProcessorByName(processor.name); if (processorWithTheSameName) { - throw generateProcessorNameAlreadyExistsError(); + throw generateError(ERROR_MESSAGES.PROCESSOR_NAME_ALREADY_EXIST, 400); } let processorId = uuid.v4(); try { - fileManager.validateJavascriptContent(processor.javascript); + let exportedFunctions = verifyJSAndGetExportedFunctions(processor.javascript); + processor.exported_functions = exportedFunctions; await databaseConnector.insertProcessor(processorId, processor); processor.id = processorId; logger.info('Processor saved successfully to database'); @@ -28,7 +27,8 @@ module.exports.createProcessor = async function (processor) { }; module.exports.getAllProcessors = async function (from, limit) { - return databaseConnector.getAllProcessors(from, limit); + let allProcessors = await databaseConnector.getAllProcessors(from, limit); + return allProcessors; }; module.exports.getProcessor = async function (processorId) { @@ -36,7 +36,7 @@ module.exports.getProcessor = async function (processorId) { if (processor) { return processor; } else { - const error = generateProcessorNotFoundError(); + const error = generateError(ERROR_MESSAGES.NOT_FOUND, 404); throw error; } }; @@ -44,7 +44,9 @@ module.exports.getProcessor = async function (processorId) { module.exports.deleteProcessor = async function (processorId) { const tests = await testsManager.getTestsByProcessorId(processorId); if (tests.length > 0) { - throw generateProcessorIsUsedByTestsError(tests.map(test => test.name)); + let testNames = tests.map(test => test.name); + let message = `${ERROR_MESSAGES.PROCESSOR_DELETION_FORBIDDEN}: ${testNames.join(', ')}`; + throw generateError(message, 409); } return databaseConnector.deleteProcessor(processorId); }; @@ -52,35 +54,44 @@ module.exports.deleteProcessor = async function (processorId) { module.exports.updateProcessor = async function (processorId, processor) { const oldProcessor = await databaseConnector.getProcessorById(processorId); if (!oldProcessor) { - throw generateProcessorNotFoundError(); + throw generateError(ERROR_MESSAGES.NOT_FOUND, 404); } if (oldProcessor.name !== processor.name) { const processorWithUpdatedName = await databaseConnector.getProcessorByName(processor.name); if (processorWithUpdatedName) { - throw generateProcessorNameAlreadyExistsError(); + throw generateError(ERROR_MESSAGES.PROCESSOR_NAME_ALREADY_EXIST, 400); } } processor.created_at = oldProcessor.created_at; - fileManager.validateJavascriptContent(processor.javascript); + let exportedFunctions = verifyJSAndGetExportedFunctions(processor.javascript); + processor.exported_functions = exportedFunctions; await databaseConnector.updateProcessor(processorId, processor); return processor; }; -function generateProcessorNotFoundError() { - const error = new Error(ERROR_MESSAGES.NOT_FOUND); - error.statusCode = 404; - return error; -} +function verifyJSAndGetExportedFunctions(src) { + let exportedFunctions; + try { + let m = new module.constructor(); + m.paths = module.paths; + m._compile(src, 'none'); + let exports = m.exports; + exportedFunctions = Object.keys(exports); + } catch (err) { + let error = generateError('javascript syntax validation failed with error: ' + err.message, 422); + throw error; + } -function generateProcessorNameAlreadyExistsError() { - const error = new Error(ERROR_MESSAGES.PROCESSOR_NAME_ALREADY_EXIST); - error.statusCode = 400; - return error; + if (exportedFunctions.length === 0) { + let error = generateError('javascript has 0 exported functions', 422); + throw error; + } + return exportedFunctions; } -function generateProcessorIsUsedByTestsError(testNames) { - const error = new Error(`${ERROR_MESSAGES.PROCESSOR_DELETION_FORBIDDEN}: ${testNames.join(', ')}`); - error.statusCode = 409; +function generateError(message, statusCode) { + const error = new Error(message); + error.statusCode = statusCode; return error; } diff --git a/src/tests/models/fileManager.js b/src/tests/models/fileManager.js index 6a32b0c19..1d72f01ee 100644 --- a/src/tests/models/fileManager.js +++ b/src/tests/models/fileManager.js @@ -1,15 +1,13 @@ 'use strict'; const uuid = require('uuid'), - request = require('request-promise-native'), - esprima = require('esprima'); + request = require('request-promise-native'); const database = require('./database'), { ERROR_MESSAGES } = require('../../common/consts'); module.exports = { saveFile, - getFile, - validateJavascriptContent + getFile }; async function downloadFile(fileUrl) { @@ -45,15 +43,3 @@ async function saveFile(fileUrl) { await database.saveFile(id, fileBase64Value); return id; } - -function validateJavascriptContent (javascriptFileContent) { - let error, errorMessage; - try { - esprima.parseScript(javascriptFileContent); - } catch (err) { - errorMessage = err.description; - error = new Error('javascript syntax validation failed with error: ' + errorMessage); - error.statusCode = 422; - throw error; - } -} \ No newline at end of file diff --git a/tests/integration-tests/jobs/createJobDocker-test.js b/tests/integration-tests/jobs/createJobDocker-test.js index cb4a17302..c07eb5512 100644 --- a/tests/integration-tests/jobs/createJobDocker-test.js +++ b/tests/integration-tests/jobs/createJobDocker-test.js @@ -146,8 +146,8 @@ describe('Create job specific docker tests', async function () { should(containers.length).eql(2); }); - it('Wait for 5 seconds to let predator runner finish', (done) => { - setTimeout(done, 5000); + it('Wait for 10 seconds to let predator runner finish', (done) => { + setTimeout(done, 10000); }); it('Delete containers', async () => { diff --git a/tests/integration-tests/processors/processors-test.js b/tests/integration-tests/processors/processors-test.js index 8a445618c..c08d06b90 100644 --- a/tests/integration-tests/processors/processors-test.js +++ b/tests/integration-tests/processors/processors-test.js @@ -108,6 +108,7 @@ describe('Processors api', function() { let getProcessorResponse = await processorRequestSender.getProcessor(processor.id, validHeaders); getProcessorResponse.statusCode.should.eql(200); should(getProcessorResponse.body).containDeep(processorData); + should(getProcessorResponse.body.exported_functions).eql(['simple']); }); it('Get non-existent processor by id', async () => { let getProcessorResponse = await processorRequestSender.getProcessor(uuid(), validHeaders); @@ -138,6 +139,7 @@ describe('Processors api', function() { }; let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders); createProcessorResponse.statusCode.should.eql(201); + createProcessorResponse.body.exported_functions.should.eql(['createAuthToken']); let deleteResponse = await processorRequestSender.deleteProcessor(createProcessorResponse.body.id); should(deleteResponse.statusCode).equal(204); @@ -145,7 +147,7 @@ describe('Processors api', function() { }); describe('PUT /v1/processors/{processor_id}', function() { it('update a processor', async function() { - const processor = generateRawJSProcessor('predator'); + const processor = generateRawJSProcessor('predator ' + uuid()); const createResponse = await processorRequestSender.createProcessor(processor, validHeaders); should(createResponse.statusCode).equal(201); const processorId = createResponse.body.id; @@ -156,6 +158,7 @@ describe('Processors api', function() { should(updateResponse.statusCode).equal(200); should(updateResponse.body.javascript).equal(processor.javascript); should(updateResponse.body.description).equal(processor.description); + should(updateResponse.body.exported_functions).eql(['add']); const deleteResponse = await processorRequestSender.deleteProcessor(processorId); should(deleteResponse.statusCode).equal(204); @@ -244,6 +247,72 @@ describe('Processors api', function() { let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders); createProcessorResponse.statusCode.should.eql(422); }); + + + it('Create processor without export functions', async () => { + const requestBody = { + name: 'authentication', + description: 'Creates authorization token and saves it in the context', + javascript: + `{ + const uuid = require('uuid/v4'); + module.exports = { + }; + + function createAuthToken(userContext, events, done) { + userContext.vars.token = uuid(); + return done(); + } + }` + }; + let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(422); + createProcessorResponse.body.message.should.eql('javascript has 0 exported functions'); + }); + + it('Create processor export function that not exists', async () => { + const requestBody = { + name: 'authentication', + description: 'Creates authorization token and saves it in the context', + javascript: + `{ + const uuid = require('uuid/v4'); + module.exports = { + hello, + }; + + function createAuthToken(userContext, events, done) { + userContext.vars.token = uuid(); + return done(); + } + }` + }; + let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(422); + createProcessorResponse.body.message.should.eql('javascript syntax validation failed with error: hello is not defined'); + }); + + it('Create processor with Unexpected token', async () => { + const requestBody = { + name: 'authentication', + description: 'Creates authorization token and saves it in the context', + javascript: + `{ + const uuid = require('uuid/v4'); + module.exports = { + hello, + }; + + function createAut) { + userContext.vars.token = uuid(); + return done(); + } + }` + }; + let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(422); + createProcessorResponse.body.message.should.eql('javascript syntax validation failed with error: Unexpected token )'); + }); }); describe('PUT /processors/{processor_id}', () => { it('processor doesn\'t exist', async function() { @@ -282,6 +351,6 @@ function generateRawJSProcessor(name) { return { name, description: 'exports a number', - javascript: 'module.exports = 5;' + javascript: 'module.exports.simple = 5;' }; } diff --git a/tests/integration-tests/runLocal.sh b/tests/integration-tests/runLocal.sh index f900c0c31..1f263a83b 100755 --- a/tests/integration-tests/runLocal.sh +++ b/tests/integration-tests/runLocal.sh @@ -4,4 +4,4 @@ LOCAL_TEST=true DATABASE_TYPE=cassandra JOB_PLATFORM=kubernetes ./tests/integrat LOCAL_TEST=true DATABASE_TYPE=mysql JOB_PLATFORM=kubernetes ./tests/integration-tests/run.sh LOCAL_TEST=true DATABASE_TYPE=sqlite JOB_PLATFORM=kubernetes ./tests/integration-tests/run.sh LOCAL_TEST=true DATABASE_TYPE=postgres JOB_PLATFORM=metronome ./tests/integration-tests/run.sh -LOCAL_TEST=true DATABASE_TYPE=sqlite JOB_PLATFORM=docker ./tests/integration-tests/run.sh \ No newline at end of file +LOCAL_TEST=true DATABASE_TYPE=sqlite JOB_PLATFORM=docker ./tests/integration-tests/run.sh diff --git a/tests/unit-tests/processors/models/processorsManager-test.js b/tests/unit-tests/processors/models/processorsManager-test.js index 02d575a11..c066a31cc 100644 --- a/tests/unit-tests/processors/models/processorsManager-test.js +++ b/tests/unit-tests/processors/models/processorsManager-test.js @@ -34,7 +34,6 @@ describe('Processor manager tests', function () { }); beforeEach(() => { - sandbox.resetHistory(); sandbox.reset(); }); @@ -85,6 +84,22 @@ describe('Processor manager tests', function () { should(e.statusCode).equal(400); } }); + + it('Should return error for js without public functions', async function () { + let exception; + const firstProcessor = { + description: 'bad processor', + name: 'mickey', + javascript: 'return 5' + }; + try { + await manager.createProcessor(firstProcessor); + } catch (e) { + exception = e; + } + should(exception.statusCode).eql(422); + should(exception.message).eql('javascript has 0 exported functions'); + }); }); describe('Delete existing processor', function () { it('Should delete processor', async function () { diff --git a/tests/unit-tests/tests/models/fileManager-test.js b/tests/unit-tests/tests/models/fileManager-test.js deleted file mode 100644 index d4ce87598..000000000 --- a/tests/unit-tests/tests/models/fileManager-test.js +++ /dev/null @@ -1,36 +0,0 @@ -'use strict'; -const should = require('should'); - -const fileManager = require('../../../../src/tests/models/fileManager'); - -describe('Javascript validation', function () { - it('Should pass javascript validation', function () { - fileManager.validateJavascriptContent(` - { - let i = 10; - i++; - console.log(i); - } - `) - }); - - it('Should fail javascript validation with error thrown', function () { - let error; - try { - fileManager.validateJavascriptContent(` - { - return 10; - - function xyz() { - console.log('xyz') - } - } - `) - } catch(e) { - error = e; - } - - should(error.statusCode).eql(422); - should(error.message).containDeep('javascript syntax validation failed with error: Illegal return statement'); - }); -});