feat(custom-js): implement GET /processors (#212)

* chore(dependencies): update cassandra-driver to v4.1.0 (#210)

* feat(custom-js): implement GET /processors (Sequelize only)

* test(processors): add tests for GET /processors (#186)

* feat(custom-js): implement GET /processors (cassandra)

* test(processors): add unit-tests for insertProcessor

src/app.js

@@ -8,7 +8,7 @@ let reportsRouter = require('./reports/routes/reportsRoute.js');
 let configRouter = require('./configManager/routes/configRoute.js');
 let dslRouter = require('./tests/routes/dslRoute.js');
 let testsRouter = require('./tests/routes/testsRoute.js');
-let processorssRouter = require('./processors/routes/processorsRoute.js');
+let processorsRouter = require('./processors/routes/processorsRoute.js');
 
 let swaggerValidator = require('express-ajv-swagger-validation');
 let audit = require('express-requests-logger');
@@ -57,7 +57,7 @@ module.exports = () => {
             app.use('/v1/dsl', dslRouter);
             app.use('/v1/tests', reportsRouter);
             app.use('/v1/tests', testsRouter);
-            app.use('/v1/processors', processorssRouter);
+            app.use('/v1/processors', processorsRouter);
 
             app.use('/', function (req, res, next) {
                 res.redirect('/ui');
@@ -76,4 +76,4 @@ module.exports = () => {
 
             return app;
         });
-};
+};

src/common/consts.js

@@ -29,4 +29,4 @@ module.exports = {
         PROMETHEUS_METRICS: 'prometheus_metrics',
         SMTP_SERVER: 'smtp_server'
     }
-};
+};

src/database/cassandra-handler/init-scripts/14__create_processors_table.cql

@@ -8,5 +8,5 @@ CREATE TABLE IF NOT EXISTS processors
     javascript         text,
     created_at timestamp,
     updated_at timestamp,
-    PRIMARY KEY (processor_id)
-);
+    PRIMARY KEY (processor_id, created_at))
+    WITH CLUSTERING ORDER BY (created_at DESC);

src/processors/controllers/processorController.js

@@ -9,4 +9,17 @@ module.exports.createProcessor = function (req, res, next) {
         .catch(function (err) {
             return next(err);
         });
-};
+};
+
+module.exports.getAllProcessors = async function (req, res, next) {
+    let { query: { from = 0, limit = 100 } } = req;
+    let processors;
+    try {
+        from = parseInt(from);
+        limit = parseInt(limit);
+        processors = await processorManager.getAllProcessors(from, limit);
+        return res.status(200).json(processors);
+    } catch (err) {
+        return next(err);
+    }
+};

src/processors/models/database/cassandra/cassandraConnector.js

@@ -1,12 +1,14 @@
 let logger = require('../../../../common/logger');
 let databaseConfig = require('../../../../config/databaseConfig');
+let _ = require('lodash');
 let client;
 
 const INSERT_PROCESSOR = 'INSERT INTO processors(processor_id, name, description, type, file_url, javascript, created_at, updated_at) values(?,?,?,?,?,?,?,?)';
-
+const GET_ALL_PROCESSORS = 'SELECT * FROM processors';
 module.exports = {
     init,
-    insertProcessor
+    insertProcessor,
+    getAllProcessors
 };
 
 let queryOptions = {
@@ -18,6 +20,11 @@ async function init(cassandraClient) {
     client = cassandraClient;
 }
 
+async function getAllProcessors(from, limit) {
+    const resultRows = await executeQuery(GET_ALL_PROCESSORS, [], {});
+    return _(resultRows).slice(from).take(limit).value();
+}
+
 function insertProcessor(processorId, processorInfo) {
     let params = [processorId, processorInfo.name, processorInfo.description, processorInfo.type, processorInfo.file_url, processorInfo.javascript, Date.now(), Date.now()];
     return executeQuery(INSERT_PROCESSOR, params, queryOptions);
@@ -35,4 +42,4 @@ function executeQuery(query, params, queryOptions) {
         logger.error(`Cassandra query failed \n ${JSON.stringify({ query, params, queryOptions })}`, exception);
         return Promise.reject(new Error('Error occurred in communication with cassandra'));
     });
-}
+}

src/processors/models/database/databaseConnector.js

@@ -1,24 +1,24 @@
-'use strict';
-
 let databaseConfig = require('../../../config/databaseConfig');
 let cassandraConnector = require('./cassandra/cassandraConnector');
 let sequelizeConnector = require('./sequelize/sequelizeConnector');
 let databaseConnector = databaseConfig.type.toLowerCase() === 'cassandra' ? cassandraConnector : sequelizeConnector;
-
 module.exports = {
     init,
-    closeConnection,
-    insertProcessor
+    getAllProcessors,
+    insertProcessor,
+    closeConnection
 };
-
 async function insertProcessor(jobId, jobInfo) {
     return databaseConnector.insertProcessor(jobId, jobInfo);
 }
-
 async function init() {
     return databaseConnector.init();
 }
 
 function closeConnection() {
     return databaseConnector.closeConnection();
-}
+}
+
+async function getAllProcessors(from, limit) {
+    return databaseConnector.getAllProcessors(from, limit);
+}

src/processors/models/database/sequelize/sequelizeConnector.js

@@ -5,6 +5,7 @@ let client;
 
 module.exports = {
     init,
+    getAllProcessors,
     insertProcessor
 };
 
@@ -28,6 +29,11 @@ async function insertProcessor(processorId, processorInfo) {
     return processor.create(params);
 }
 
+async function getAllProcessors(from, limit) {
+    const processorsModel = client.model('processor');
+    return processorsModel.findAll({ offset: from, limit, order: [['created_at', 'DESC']] });
+}
+
 async function initSchemas() {
     const processorsFiles = client.define('processor', {
         processor_id: {

src/processors/models/processorsManager.js

@@ -21,4 +21,8 @@ module.exports.createProcessor = async function (processor) {
         logger.error(error, 'Error occurred trying to create new processor');
         return Promise.reject(error);
     }
-};
+};
+
+module.exports.getAllProcessors = async function(from, limit) {
+    return databaseConnector.getAllProcessors(from, limit);
+};

src/processors/routes/processorsRoute.js

@@ -6,6 +6,7 @@ let router = express.Router();
 
 let processors = require('../controllers/processorController');
 
+router.get('/', swaggerValidator.validate, processors.getAllProcessors);
 router.post('/', swaggerValidator.validate, processors.createProcessor);
 
-module.exports = router;
+module.exports = router;

tests/integration-tests/processors/helpers/requestCreator.js

@@ -1,11 +1,15 @@
 
 const request = require('supertest'),
     expressApp = require('../../../../src/app');
+
 let app;
+
 module.exports = {
     init,
-    createProcessor
+    createProcessor,
+    getProcessors
 };
+
 async function init() {
     try {
         app = await expressApp();
@@ -22,4 +26,13 @@ function createProcessor(body, headers) {
         .expect(function(res){
             return res;
         });
-}
+}
+
+function getProcessors(from, limit) {
+    return request(app).get('/v1/processors')
+        .query({ from, limit })
+        .set({ 'Content-Type': 'application/json' })
+        .expect(function (res) {
+            return res;
+        });
+}

tests/integration-tests/processors/processors-test.js

@@ -9,20 +9,77 @@ describe('Processors api', function() {
         await requestSender.init();
     });
 
-    describe('Good requests', function() {
+    describe('Good requests', async function() {
+        describe('GET /v1/processors', async function () {
+            let numberOfProcessorsToInsert = 101;
+            let jsProcessorsArr = [];
+            let processorsInserted = [];
+            before(async function() {
+                for (let i = 0; i < numberOfProcessorsToInsert; i++) {
+                    const processor = generateRawJSProcessor(i.toString());
+                    jsProcessorsArr.push(processor);
+                    const processorRes = await requestSender.createProcessor(processor, validHeaders);
+                    processorsInserted.push(processorRes);
+                }
+            });
+            it('Check default paging values (from = 0, limit = 100)', async function() {
+                let getProcessorsResponse = await requestSender.getProcessors();
+
+                should(getProcessorsResponse.statusCode).equal(200);
+
+                const processors = getProcessorsResponse.body;
+                should(processors.length).equal(100);
+            });
+
+            it('Get a page', async function() {
+                const from = 25, limit = 50;
+                let getProcessorsResponse = await requestSender.getProcessors(from, limit);
+
+                should(getProcessorsResponse.statusCode).equal(200);
+
+                const processors = getProcessorsResponse.body;
+                should(processors.length).equal(limit);
+            });
+
+            it('Validate from parameter starts from the correct index', async function() {
+                const from = 0, limit = 50;
+                let page1Response = await requestSender.getProcessors(from, limit);
+                let page2Response = await requestSender.getProcessors(from + 1, limit);
+
+                should(page1Response.statusCode).equal(200);
+                should(page2Response.statusCode).equal(200);
+
+                const page1Processors = page1Response.body;
+                const page2Processors = page2Response.body;
+                should(page1Processors[1]).deepEqual(page2Processors[0]);
+            });
+
+            it('Get a page with limit > # of processors', async function() {
+                const from = 0, limit = 1000;
+                let getProcessorsResponse = await requestSender.getProcessors(from, limit);
+
+                should(getProcessorsResponse.statusCode).equal(200);
+
+                const processors = getProcessorsResponse.body;
+                should(processors.length).greaterThanOrEqual(101);
+            });
+            after(async function() {
+                // TODO: when DELETE /processors is implemented, use processorsInserted to empty the table.
+            });
+        });
         it('Create processor with type file_download', async () => {
             nock('https://authentication.predator.dev').get('/?dl=1').reply(200,
-                `{ 
-                     const uuid = require('uuid/v4');
-                     module.exports = {
-                       createAuthToken
-                     };
+                `{
+                        const uuid = require('uuid/v4');
+                        module.exports = {
+                        createAuthToken
+                        };
 
-                     function createAuthToken(userContext, events, done) {
-                       userContext.vars.token = uuid();
-                       return done();
-                     }
-                 }`
+                        function createAuthToken(userContext, events, done) {
+                        userContext.vars.token = uuid();
+                        return done();
+                        }
+                    }`
             );
 
             const requestBody = {
@@ -41,17 +98,17 @@ describe('Processors api', function() {
                 description: 'Creates authorization token and saves it in the context',
                 type: 'raw_javascript',
                 javascript:
-                    `{ 
-                     const uuid = require('uuid/v4');
-                     module.exports = {
-                       createAuthToken
-                     };
+                    `{
+                        const uuid = require('uuid/v4');
+                        module.exports = {
+                        createAuthToken
+                        };
 
-                     function createAuthToken(userContext, events, done) {
-                       userContext.vars.token = uuid();
-                       return done();
-                     }
-                 }`
+                        function createAuthToken(userContext, events, done) {
+                        userContext.vars.token = uuid();
+                        return done();
+                        }
+                    }`
             };
             let createProcessorResponse = await requestSender.createProcessor(requestBody, validHeaders);
             createProcessorResponse.statusCode.should.eql(201);
@@ -107,17 +164,17 @@ describe('Processors api', function() {
 
         it('Create processor with type file_download and invalid js syntax', async () => {
             nock('https://authentication.predator.dev').get('/?dl=1').reply(200,
-                `{ 
+                `{
                      const uuid = require('uuid/v4');
                      module.exports = {
                        createAuthToken
                      };
-                     
+
                      function createAuthToken(userContext, events, done) {
                        userContext.vars.token = uuid();
                        return done();
                      }
-                     
+
                      this is not valid javascript
                  }`
             );
@@ -131,4 +188,13 @@ describe('Processors api', function() {
             createProcessorResponse.statusCode.should.eql(422);
         });
     });
-});
+});
+
+function generateRawJSProcessor(name) {
+    return {
+        name,
+        description: 'exports a number',
+        type: 'raw_javascript',
+        javascript: 'module.exports = 5;'
+    };
+}