feat(processors): adding exported functions to processors (#255)

* feat(processors): adding exported functions to processors
Zooz · Jan 14, 2020 · 76f0344 · 76f0344
1 parent d1d358d
commit 76f0344
Show file tree

Hide file tree

Showing 13 changed files with 167 additions and 89 deletions.
diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml
@@ -1326,6 +1326,7 @@ components:
       properties:
         name:
           type: string
+          minLength: 1
           description: The name of the test.
           example: Order from Pet Store
         processor_file_url:
@@ -1758,12 +1759,19 @@ components:
           readOnly: true
         name:
           type: string
+          minLength: 1
           description: The name of the processor.
           example: Custom javascript for logging
         description:
           type: string
           description: A description of the processor.
           example: logs every error (5xx).
+        exported_functions:
+          type: array
+          readOnly: true
+          description: Names of all exported function in the javascript file
+          items:
+            type: string
         updated_at:
           type: string
           format: date-time

diff --git a/package.json b/package.json
@@ -44,7 +44,6 @@
     "copy-dir": "^0.3.0",
     "cron": "^1.7.1",
     "dockerode": "^2.5.8",
-    "esprima": "^4.0.1",
     "express": "^4.17.1",
     "express-ajv-swagger-validation": "^0.9.0",
     "express-easy-zip": "^1.1.4",

diff --git a/src/database/cassandra-handler/init-scripts/17__processors_exported_functions.cql b/src/database/cassandra-handler/init-scripts/17__processors_exported_functions.cql
@@ -0,0 +1 @@
+ALTER TABLE processors ADD exported_functions list<text>;
diff --git a/src/database/sequlize-handler/migrations/03_tests_processor_exported_functions.js b/src/database/sequlize-handler/migrations/03_tests_processor_exported_functions.js
@@ -0,0 +1,15 @@
+const Sequelize = require('sequelize');
+
+module.exports.up = async (query, DataTypes) => {
+    let testsTable = await query.describeTable('processors');
+
+    if (!testsTable.exported_functions) {
+        await query.addColumn(
+            'processors', 'exported_functions',
+            Sequelize.DataTypes.STRING);
+    }
+};
+
+module.exports.down = async (query, DataTypes) => {
+    await query.removeColumn('processors', 'exported_functions');
+};
diff --git a/src/processors/models/database/cassandra/cassandraConnector.js b/src/processors/models/database/cassandra/cassandraConnector.js
@@ -3,11 +3,11 @@ let databaseConfig = require('../../../../config/databaseConfig');
 let _ = require('lodash');
 let client;
 
-const INSERT_PROCESSOR = 'INSERT INTO processors(id, name, description, javascript, created_at, updated_at) values(?,?,?,?,?,?)';
+const INSERT_PROCESSOR = 'INSERT INTO processors(id, name, description, javascript, exported_functions, created_at, updated_at) values(?,?,?,?,?,?,?)';
 const GET_ALL_PROCESSORS = 'SELECT * FROM processors';
 const GET_PROCESSOR_BY_ID = 'SELECT * FROM processors WHERE id=?';
 const DELETE_PROCESSOR = 'DELETE FROM processors WHERE id=?';
-const UPDATE_PROCESSOR = 'UPDATE processors SET name=?, description=?, javascript=?, updated_at=? WHERE id=? AND created_at=? IF EXISTS';
+const UPDATE_PROCESSOR = 'UPDATE processors SET name=?, description=?, javascript=?, exported_functions=?, updated_at=? WHERE id=? AND created_at=? IF EXISTS';
 
 const INSERT_PROCESSOR_MAPPING = 'INSERT INTO processors_mapping(name, id) VALUES(?, ?)';
 const DELETE_PROCESSOR_MAPPING = 'DELETE FROM processors_mapping WHERE name=?';
@@ -72,7 +72,7 @@ async function deleteProcessor(processorId) {
 }
 
 async function insertProcessor(processorId, processorInfo) {
-    let params = [processorId, processorInfo.name, processorInfo.description, processorInfo.javascript, Date.now(), Date.now()];
+    let params = [processorId, processorInfo.name, processorInfo.description, processorInfo.javascript, processorInfo.exported_functions, Date.now(), Date.now()];
     let mappingParams = [processorInfo.name, processorId];
     const [processor] = await Promise.all([
         executeQuery(INSERT_PROCESSOR, params, queryOptions),
@@ -82,9 +82,9 @@ async function insertProcessor(processorId, processorInfo) {
 }
 
 async function updateProcessor(processorId, updatedProcessor) {
-    const { name, description, javascript, created_at: createdAt } = updatedProcessor;
+    const { name, description, javascript, exported_functions, created_at: createdAt } = updatedProcessor;
     const processor = await getProcessorById(processorId);
-    const params = [ name, description, javascript, Date.now(), processorId, createdAt.getTime() ];
+    const params = [ name, description, javascript, exported_functions, Date.now(), processorId, createdAt.getTime() ];
     return Promise.all([
         executeQuery(UPDATE_PROCESSOR, params, queryOptions),
         executeQuery(INSERT_PROCESSOR_MAPPING, [updatedProcessor.name, processorId]),

diff --git a/src/processors/models/database/sequelize/sequelizeConnector.js b/src/processors/models/database/sequelize/sequelizeConnector.js
@@ -25,6 +25,7 @@ async function insertProcessor(processorId, processorInfo) {
         name: processorInfo.name,
         description: processorInfo.description,
         javascript: processorInfo.javascript,
+        exported_functions: processorInfo.exported_functions,
         created_at: Date.now(),
         updated_at: Date.now()
     };
@@ -67,8 +68,8 @@ async function deleteProcessor(processorId) {
 
 async function updateProcessor(processorId, updatedProcessor) {
     const processorsModel = client.model('processor');
-    const { name, description, javascript } = updatedProcessor;
-    return processorsModel.update({ name, description, javascript, updated_at: Date.now() }, { where: { id: processorId } });
+    const { name, description, javascript, exported_functions } = updatedProcessor;
+    return processorsModel.update({ name, description, javascript, updated_at: Date.now(), exported_functions }, { where: { id: processorId } });
 }
 
 async function initSchemas() {
@@ -91,6 +92,15 @@ async function initSchemas() {
         },
         updated_at: {
             type: Sequelize.DataTypes.DATE
+        },
+        exported_functions: {
+            type: Sequelize.DataTypes.STRING,
+            get: function() {
+                return JSON.parse(this.getDataValue('exported_functions'));
+            },
+            set: function(val) {
+                return this.setDataValue('exported_functions', JSON.stringify(val));
+            }
         }
     });
     await processorsFiles.sync();

diff --git a/src/processors/models/processorsManager.js b/src/processors/models/processorsManager.js
@@ -4,19 +4,18 @@ const uuid = require('uuid');
 
 const logger = require('../../common/logger'),
     databaseConnector = require('./database/databaseConnector'),
-    fileManager = require('../../tests/models/fileManager.js'),
+    testsManager = require('../../tests/models/manager'),
     { ERROR_MESSAGES } = require('../../common/consts');
 
-let testsManager = require('../../tests/models/manager');
-
 module.exports.createProcessor = async function (processor) {
     const processorWithTheSameName = await databaseConnector.getProcessorByName(processor.name);
     if (processorWithTheSameName) {
-        throw generateProcessorNameAlreadyExistsError();
+        throw generateError(ERROR_MESSAGES.PROCESSOR_NAME_ALREADY_EXIST, 400);
     }
     let processorId = uuid.v4();
     try {
-        fileManager.validateJavascriptContent(processor.javascript);
+        let exportedFunctions = verifyJSAndGetExportedFunctions(processor.javascript);
+        processor.exported_functions = exportedFunctions;
         await databaseConnector.insertProcessor(processorId, processor);
         processor.id = processorId;
         logger.info('Processor saved successfully to database');
@@ -28,59 +27,71 @@ module.exports.createProcessor = async function (processor) {
 };
 
 module.exports.getAllProcessors = async function (from, limit) {
-    return databaseConnector.getAllProcessors(from, limit);
+    let allProcessors = await databaseConnector.getAllProcessors(from, limit);
+    return allProcessors;
 };
 
 module.exports.getProcessor = async function (processorId) {
     const processor = await databaseConnector.getProcessorById(processorId);
     if (processor) {
         return processor;
     } else {
-        const error = generateProcessorNotFoundError();
+        const error = generateError(ERROR_MESSAGES.NOT_FOUND, 404);
         throw error;
     }
 };
 
 module.exports.deleteProcessor = async function (processorId) {
     const tests = await testsManager.getTestsByProcessorId(processorId);
     if (tests.length > 0) {
-        throw generateProcessorIsUsedByTestsError(tests.map(test => test.name));
+        let testNames = tests.map(test => test.name);
+        let message = `${ERROR_MESSAGES.PROCESSOR_DELETION_FORBIDDEN}: ${testNames.join(', ')}`;
+        throw generateError(message, 409);
     }
     return databaseConnector.deleteProcessor(processorId);
 };
 
 module.exports.updateProcessor = async function (processorId, processor) {
     const oldProcessor = await databaseConnector.getProcessorById(processorId);
     if (!oldProcessor) {
-        throw generateProcessorNotFoundError();
+        throw generateError(ERROR_MESSAGES.NOT_FOUND, 404);
     }
     if (oldProcessor.name !== processor.name) {
         const processorWithUpdatedName = await databaseConnector.getProcessorByName(processor.name);
         if (processorWithUpdatedName) {
-            throw generateProcessorNameAlreadyExistsError();
+            throw generateError(ERROR_MESSAGES.PROCESSOR_NAME_ALREADY_EXIST, 400);
         }
     }
 
     processor.created_at = oldProcessor.created_at;
-    fileManager.validateJavascriptContent(processor.javascript);
+    let exportedFunctions = verifyJSAndGetExportedFunctions(processor.javascript);
+    processor.exported_functions = exportedFunctions;
     await databaseConnector.updateProcessor(processorId, processor);
     return processor;
 };
 
-function generateProcessorNotFoundError() {
-    const error = new Error(ERROR_MESSAGES.NOT_FOUND);
-    error.statusCode = 404;
-    return error;
-}
+function verifyJSAndGetExportedFunctions(src) {
+    let exportedFunctions;
+    try {
+        let m = new module.constructor();
+        m.paths = module.paths;
+        m._compile(src, 'none');
+        let exports = m.exports;
+        exportedFunctions = Object.keys(exports);
+    } catch (err) {
+        let error = generateError('javascript syntax validation failed with error: ' + err.message, 422);
+        throw error;
+    }
 
-function generateProcessorNameAlreadyExistsError() {
-    const error = new Error(ERROR_MESSAGES.PROCESSOR_NAME_ALREADY_EXIST);
-    error.statusCode = 400;
-    return error;
+    if (exportedFunctions.length === 0) {
+        let error = generateError('javascript has 0 exported functions', 422);
+        throw error;
+    }
+    return exportedFunctions;
 }
 
-function generateProcessorIsUsedByTestsError(testNames) {
-    const error = new Error(`${ERROR_MESSAGES.PROCESSOR_DELETION_FORBIDDEN}: ${testNames.join(', ')}`);
-    error.statusCode = 409;
+function generateError(message, statusCode) {
+    const error = new Error(message);
+    error.statusCode = statusCode;
     return error;
 }
diff --git a/src/tests/models/fileManager.js b/src/tests/models/fileManager.js
@@ -1,15 +1,13 @@
 'use strict';
 const uuid = require('uuid'),
-    request = require('request-promise-native'),
-    esprima = require('esprima');
+    request = require('request-promise-native');
 
 const database = require('./database'),
     { ERROR_MESSAGES } = require('../../common/consts');
 
 module.exports = {
     saveFile,
-    getFile,
-    validateJavascriptContent
+    getFile
 };
 
 async function downloadFile(fileUrl) {
@@ -45,15 +43,3 @@ async function saveFile(fileUrl) {
     await database.saveFile(id, fileBase64Value);
     return id;
 }
-
-function validateJavascriptContent (javascriptFileContent) {
-    let error, errorMessage;
-    try {
-        esprima.parseScript(javascriptFileContent);
-    } catch (err) {
-        errorMessage = err.description;
-        error = new Error('javascript syntax validation failed with error: ' + errorMessage);
-        error.statusCode = 422;
-        throw error;
-    }
-}
diff --git a/tests/integration-tests/jobs/createJobDocker-test.js b/tests/integration-tests/jobs/createJobDocker-test.js
@@ -146,8 +146,8 @@ describe('Create job specific docker tests', async function () {
                         should(containers.length).eql(2);
                     });
 
-                    it('Wait for 5 seconds to let predator runner finish', (done) => {
-                        setTimeout(done, 5000);
+                    it('Wait for 10 seconds to let predator runner finish', (done) => {
+                        setTimeout(done, 10000);
                     });
 
                     it('Delete containers', async () => {

diff --git a/tests/integration-tests/processors/processors-test.js b/tests/integration-tests/processors/processors-test.js
@@ -108,6 +108,7 @@ describe('Processors api', function() {
                 let getProcessorResponse = await processorRequestSender.getProcessor(processor.id, validHeaders);
                 getProcessorResponse.statusCode.should.eql(200);
                 should(getProcessorResponse.body).containDeep(processorData);
+                should(getProcessorResponse.body.exported_functions).eql(['simple']);
             });
             it('Get non-existent processor by id', async () => {
                 let getProcessorResponse = await processorRequestSender.getProcessor(uuid(), validHeaders);
@@ -138,14 +139,15 @@ describe('Processors api', function() {
                 };
                 let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders);
                 createProcessorResponse.statusCode.should.eql(201);
+                createProcessorResponse.body.exported_functions.should.eql(['createAuthToken']);
 
                 let deleteResponse = await processorRequestSender.deleteProcessor(createProcessorResponse.body.id);
                 should(deleteResponse.statusCode).equal(204);
             });
         });
         describe('PUT /v1/processors/{processor_id}', function() {
             it('update a processor', async function() {
-                const processor = generateRawJSProcessor('predator');
+                const processor = generateRawJSProcessor('predator ' + uuid());
                 const createResponse = await processorRequestSender.createProcessor(processor, validHeaders);
                 should(createResponse.statusCode).equal(201);
                 const processorId = createResponse.body.id;
@@ -156,6 +158,7 @@ describe('Processors api', function() {
                 should(updateResponse.statusCode).equal(200);
                 should(updateResponse.body.javascript).equal(processor.javascript);
                 should(updateResponse.body.description).equal(processor.description);
+                should(updateResponse.body.exported_functions).eql(['add']);
 
                 const deleteResponse = await processorRequestSender.deleteProcessor(processorId);
                 should(deleteResponse.statusCode).equal(204);
@@ -244,6 +247,72 @@ describe('Processors api', function() {
                 let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders);
                 createProcessorResponse.statusCode.should.eql(422);
             });
+
+
+            it('Create processor without export functions', async () => {
+                const requestBody = {
+                    name: 'authentication',
+                    description: 'Creates authorization token and saves it in the context',
+                    javascript:
+                        `{
+                        const uuid = require('uuid/v4');
+                        module.exports = {
+                        };
+
+                        function createAuthToken(userContext, events, done) {
+                        userContext.vars.token = uuid();
+                        return done();
+                        }
+                    }`
+                };
+                let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders);
+                createProcessorResponse.statusCode.should.eql(422);
+                createProcessorResponse.body.message.should.eql('javascript has 0 exported functions');
+            });
+
+            it('Create processor export function that not exists', async () => {
+                const requestBody = {
+                    name: 'authentication',
+                    description: 'Creates authorization token and saves it in the context',
+                    javascript:
+                        `{
+                        const uuid = require('uuid/v4');
+                        module.exports = {
+                        hello,
+                        };
+
+                        function createAuthToken(userContext, events, done) {
+                        userContext.vars.token = uuid();
+                        return done();
+                        }
+                    }`
+                };
+                let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders);
+                createProcessorResponse.statusCode.should.eql(422);
+                createProcessorResponse.body.message.should.eql('javascript syntax validation failed with error: hello is not defined');
+            });
+
+            it('Create processor with Unexpected token', async () => {
+                const requestBody = {
+                    name: 'authentication',
+                    description: 'Creates authorization token and saves it in the context',
+                    javascript:
+                        `{
+                        const uuid = require('uuid/v4');
+                        module.exports = {
+                        hello,
+                        };
+
+                        function createAut) {
+                        userContext.vars.token = uuid();
+                        return done();
+                        }
+                    }`
+                };
+                let createProcessorResponse = await processorRequestSender.createProcessor(requestBody, validHeaders);
+                createProcessorResponse.statusCode.should.eql(422);
+                createProcessorResponse.body.message.should.eql('javascript syntax validation failed with error: Unexpected token )');
+            });
         });
         describe('PUT /processors/{processor_id}', () => {
             it('processor doesn\'t exist', async function() {
@@ -282,6 +351,6 @@ function generateRawJSProcessor(name) {
     return {
         name,
         description: 'exports a number',
-        javascript: 'module.exports = 5;'
+        javascript: 'module.exports.simple = 5;'
     };
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ALTER TABLE processors ADD exported_functions list<text>;