WIP: feat(custom-js): implement POST /processors (#213)

* docs(image): fix outdated image in docs (#214)

* chore(ci/cd): remove circleci docker_layer_caching (#218)

* feat(api): add api spec for processors resource (#200)

* feat(api): add api spec for processors resource

* feat(api): add api spec for processors resource

closes #185

fix #185

* fix(typo): typo date_time to date-time

* fix(typo): typo date_time to date-time

* chore(openapi3): code review comments addressed

* docs(api): add download endpoint for processor files

* chore(dependencies): update cassandra-driver to v4.1.0 (#210)

* chore: fix rebase to master

* feat(custom-js): implement POST /processors

* test(processors): add tests for POST /processors (#217)

* test(processors): add tests for POST /processors

* style(processors): fix processors cassandra connector name

* feat(processors): add js validation on POST /processors

* fix(processors): add updated_at/created_at to processors resource

* test(processors): add validateJavascriptContent unit-tests

* test(create-test): fix processor file download error message in test

docs/openapi3.yaml

@@ -2012,6 +2012,9 @@ components:
           description: Javascript as a string.
     processor_type:
       type: string
+      enum:
+        - raw_javascript
+        - file_download
       description: |
         The type of the processor resource. Can be one of the following:
         * `raw_javascript`: Raw javascript string that will be stored and persisted as it is.

package.json

@@ -44,6 +44,7 @@
     "copy-dir": "^0.3.0",
     "cron": "^1.7.1",
     "dockerode": "^2.5.8",
+    "esprima": "^4.0.1",
     "express": "^4.17.1",
     "express-ajv-swagger-validation": "^0.9.0",
     "express-easy-zip": "^1.1.4",

src/app.js

@@ -8,6 +8,7 @@ let reportsRouter = require('./reports/routes/reportsRoute.js');
 let configRouter = require('./configManager/routes/configRoute.js');
 let dslRouter = require('./tests/routes/dslRoute.js');
 let testsRouter = require('./tests/routes/testsRoute.js');
+let processorssRouter = require('./processors/routes/processorsRoute.js');
 
 let swaggerValidator = require('express-ajv-swagger-validation');
 let audit = require('express-requests-logger');
@@ -56,6 +57,7 @@ module.exports = () => {
             app.use('/v1/dsl', dslRouter);
             app.use('/v1/tests', reportsRouter);
             app.use('/v1/tests', testsRouter);
+            app.use('/v1/processors', processorssRouter);
 
             app.use('/', function (req, res, next) {
                 res.redirect('/ui');

src/common/consts.js

@@ -1,6 +1,9 @@
 module.exports = {
     TEST_TYPE_BASIC: 'basic',
     TEST_TYPE_DSL: 'dsl',
+    PROCESSOR_TYPE_FILE_DOWNLOAD: 'file_download',
+    PROCESSOR_TYPE_RAW_JAVASCRIPT: 'raw_javascript',
+
     ERROR_MESSAGES: {
         NOT_FOUND: 'Not found',
         DSL_DEF_ALREADY_EXIST: 'Definition already exists'

src/database/cassandra-handler/cassandra.js

@@ -4,6 +4,7 @@ const schedulerCassandraConnector = require('../../jobs/models/database/cassandr
 const reportsCassandraConnector = require('../../reports/models/database/cassandra/cassandraConnector');
 const testsCassandraConnector = require('../../tests/models/database/cassandra/cassandraConnector');
 const configCassandraConnector = require('../../configManager/models/database/cassandra/cassandraConnector');
+const processorsCassandraConnector = require('../../processors/models/database/cassandra/cassandraConnector');
 const databaseConfig = require('../../config/databaseConfig');
 const cassandraMigration = require('./cassandraMigration');
 const logger = require('../../common/logger');
@@ -17,6 +18,7 @@ module.exports.init = async () => {
     await schedulerCassandraConnector.init(cassandraClient);
     await testsCassandraConnector.init(cassandraClient);
     await configCassandraConnector.init(cassandraClient);
+    await processorsCassandraConnector.init(cassandraClient);
     logger.info('cassandra client initialized');
 };
 

src/database/cassandra-handler/init-scripts/14__create_processors_table.cql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS processors
+(
+    processor_id       uuid,
+    name               text,
+    description        text,
+    type               text,
+    file_url           text,
+    javascript         text,
+    created_at timestamp,
+    updated_at timestamp,
+    PRIMARY KEY (processor_id)
+);

src/database/sequlize-handler/sequlize.js

@@ -6,6 +6,7 @@ const schedulerSequlizeConnector = require('../../jobs/models/database/sequelize
 const reportsSequlizeConnector = require('../../reports/models/database/sequelize/sequelizeConnector');
 const testsSequlizeConnector = require('../../tests/models/database/sequelize/sequelizeConnector');
 const configSequlizeConnector = require('../../configManager/models/database/sequelize/sequelizeConnector');
+const processorsSequlizeConnector = require('../../processors/models/database/sequelize/sequelizeConnector');
 const logger = require('../../../src/common/logger');
 const databaseConfig = require('../../config/databaseConfig');
 const Sequelize = require('sequelize');
@@ -17,6 +18,7 @@ module.exports.init = async () => {
     await reportsSequlizeConnector.init(sequlizeClient);
     await testsSequlizeConnector.init(sequlizeClient);
     await configSequlizeConnector.init(sequlizeClient);
+    await processorsSequlizeConnector.init(sequlizeClient);
     await runSequlizeMigrations();
 };
 

src/processors/controllers/processorController.js

@@ -0,0 +1,12 @@
+'use strict';
+let processorManager = require('../models/processorsManager');
+
+module.exports.createProcessor = function (req, res, next) {
+    return processorManager.createProcessor(req.body)
+        .then(function (result) {
+            return res.status(201).json(result);
+        })
+        .catch(function (err) {
+            return next(err);
+        });
+};

src/processors/models/database/cassandra/cassandraConnector.js

@@ -0,0 +1,38 @@
+let logger = require('../../../../common/logger');
+let databaseConfig = require('../../../../config/databaseConfig');
+let client;
+
+const INSERT_PROCESSOR = 'INSERT INTO processors(processor_id, name, description, type, file_url, javascript, created_at, updated_at) values(?,?,?,?,?,?,?,?)';
+
+module.exports = {
+    init,
+    insertProcessor
+};
+
+let queryOptions = {
+    consistency: databaseConfig.cassandraConsistency,
+    prepare: true
+};
+
+async function init(cassandraClient) {
+    client = cassandraClient;
+}
+
+function insertProcessor(processorId, processorInfo) {
+    let params = [processorId, processorInfo.name, processorInfo.description, processorInfo.type, processorInfo.file_url, processorInfo.javascript, Date.now(), Date.now()];
+    return executeQuery(INSERT_PROCESSOR, params, queryOptions);
+}
+
+function executeQuery(query, params, queryOptions) {
+    return client.execute(query, params, { prepare: true }, queryOptions).then((result) => {
+        logger.trace('Query result', {
+            query: query,
+            params: params,
+            rows_returned: result.rowLength
+        });
+        return Promise.resolve(result.rows ? result.rows : []);
+    }).catch((exception) => {
+        logger.error(`Cassandra query failed \n ${JSON.stringify({ query, params, queryOptions })}`, exception);
+        return Promise.reject(new Error('Error occurred in communication with cassandra'));
+    });
+}

src/processors/models/database/databaseConnector.js

@@ -0,0 +1,24 @@
+'use strict';
+
+let databaseConfig = require('../../../config/databaseConfig');
+let cassandraConnector = require('./cassandra/cassandraConnector');
+let sequelizeConnector = require('./sequelize/sequelizeConnector');
+let databaseConnector = databaseConfig.type.toLowerCase() === 'cassandra' ? cassandraConnector : sequelizeConnector;
+
+module.exports = {
+    init,
+    closeConnection,
+    insertProcessor
+};
+
+async function insertProcessor(jobId, jobInfo) {
+    return databaseConnector.insertProcessor(jobId, jobInfo);
+}
+
+async function init() {
+    return databaseConnector.init();
+}
+
+function closeConnection() {
+    return databaseConnector.closeConnection();
+}

src/processors/models/database/sequelize/sequelizeConnector.js

@@ -0,0 +1,60 @@
+'use strict';
+
+const Sequelize = require('sequelize');
+let client;
+
+module.exports = {
+    init,
+    insertProcessor
+};
+
+async function init(sequelizeClient) {
+    client = sequelizeClient;
+    await initSchemas();
+}
+
+async function insertProcessor(processorId, processorInfo) {
+    const processor = client.model('processor');
+    let params = {
+        processor_id: processorId,
+        name: processorInfo.name,
+        description: processorInfo.description,
+        type: processorInfo.type,
+        file_url: processorInfo.file_url,
+        javascript: processorInfo.javascript,
+        created_at: Date.now(),
+        updated_at: Date.now()
+    };
+    return processor.create(params);
+}
+
+async function initSchemas() {
+    const processorsFiles = client.define('processor', {
+        processor_id: {
+            type: Sequelize.DataTypes.UUID,
+            primaryKey: true
+        },
+        name: {
+            type: Sequelize.DataTypes.TEXT('medium')
+        },
+        description: {
+            type: Sequelize.DataTypes.TEXT('long')
+        },
+        type: {
+            type: Sequelize.DataTypes.TEXT('medium')
+        },
+        file_url: {
+            type: Sequelize.DataTypes.TEXT('long')
+        },
+        javascript: {
+            type: Sequelize.DataTypes.TEXT('long')
+        },
+        created_at: {
+            type: Sequelize.DataTypes.DATE
+        },
+        updated_at: {
+            type: Sequelize.DataTypes.DATE
+        }
+    });
+    await processorsFiles.sync();
+}

src/processors/models/processorsManager.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const uuid = require('uuid');
+
+const logger = require('../../common/logger'),
+    databaseConnector = require('./database/databaseConnector'),
+    common = require('../../common/consts.js'),
+    fileManager = require('../../tests/models/fileManager.js');
+
+module.exports.createProcessor = async function (processor) {
+    let processorId = uuid.v4();
+    try {
+        if (processor.type === common.PROCESSOR_TYPE_FILE_DOWNLOAD) {
+            processor.javascript = await fileManager.downloadFile(processor.file_url);
+        }
+        fileManager.validateJavascriptContent(processor.javascript);
+        await databaseConnector.insertProcessor(processorId, processor);
+        logger.info('Processor saved successfully to database');
+        return processor;
+    } catch (error) {
+        logger.error(error, 'Error occurred trying to create new processor');
+        return Promise.reject(error);
+    }
+};

src/processors/routes/processorsRoute.js

@@ -0,0 +1,11 @@
+'use strict';
+
+let swaggerValidator = require('express-ajv-swagger-validation');
+let express = require('express');
+let router = express.Router();
+
+let processors = require('../controllers/processorController');
+
+router.post('/', swaggerValidator.validate, processors.createProcessor);
+
+module.exports = router;

src/tests/models/fileManager.js

@@ -1,12 +1,16 @@
 'use strict';
-const database = require('./database'),
-    uuid = require('uuid'),
+const uuid = require('uuid'),
     request = require('request-promise-native'),
+    esprima = require('esprima');
+
+const database = require('./database'),
     { ERROR_MESSAGES } = require('../../common/consts');
 
 module.exports = {
     createFileFromUrl,
-    getFile
+    downloadFile,
+    getFile,
+    validateJavascriptContent
 };
 async function createFileFromUrl(testRawData) {
     if (testRawData['processor_file_url']) {
@@ -15,16 +19,16 @@ async function createFileFromUrl(testRawData) {
     }
     return undefined;
 }
+
 async function downloadFile(fileUrl) {
     const options = {
         url: fileUrl
     };
     try {
         const response = await request.get(options);
-        const base64Value = Buffer.from(response).toString('base64');
-        return base64Value;
+        return response;
     } catch (err) {
-        const errMsg = 'Error to read file, throw exception: ' + err;
+        const errMsg = 'Error to download file: ' + err;
         const error = new Error(errMsg);
         error.statusCode = 422;
         throw error;
@@ -45,6 +49,19 @@ async function getFile(fileId) {
 async function saveFile(fileUrl) {
     const id = uuid();
     const fileToSave = await downloadFile(fileUrl);
-    await database.saveFile(id, fileToSave);
+    const fileBase64Value = Buffer.from(fileToSave).toString('base64');
+    await database.saveFile(id, fileBase64Value);
     return id;
 }
+
+function validateJavascriptContent (javascriptFileContent) {
+    let error, errorMessage;
+    try {
+        esprima.parseScript(javascriptFileContent);
+    } catch (err) {
+        errorMessage = err.description;
+        error = new Error('javascript syntax validation failed with error: ' + errorMessage);
+        error.statusCode = 422;
+        throw error;
+    }
+}

tests/integration-tests/processors/helpers/requestCreator.js

@@ -0,0 +1,25 @@
+
+const request = require('supertest'),
+    expressApp = require('../../../../src/app');
+let app;
+module.exports = {
+    init,
+    createProcessor
+};
+async function init() {
+    try {
+        app = await expressApp();
+    } catch (err){
+        console.log(err);
+        process.exit(1);
+    }
+}
+
+function createProcessor(body, headers) {
+    return request(app).post('/v1/processors')
+        .send(body)
+        .set(headers)
+        .expect(function(res){
+            return res;
+        });
+}