API changes

terascope · Nov 8, 2018 · 112f191 · 112f191
1 parent 6016a58
commit 112f191
Show file tree

Hide file tree

Showing 7 changed files with 439 additions and 433 deletions.
diff --git a/packages/teraslice/lib/cluster/services/api.js b/packages/teraslice/lib/cluster/services/api.js
diff --git a/packages/teraslice/lib/cluster/services/assets.js b/packages/teraslice/lib/cluster/services/assets.js
@@ -5,7 +5,7 @@ const _ = require('lodash');
 const express = require('express');
 const parseError = require('@terascope/error-parser');
 const makeAssetsStore = require('../storage/assets');
-const { makeTable, handleError } = require('../../utils/api_utils');
+const { makeTable, handleRequest, getSearchOptions } = require('../../utils/api_utils');
 
 module.exports = function module(context) {
     const logger = context.apis.foundation.makeLogger({ module: 'assets_service' });
@@ -14,8 +14,16 @@ module.exports = function module(context) {
     let assetsStore;
     let running = false;
 
+    app.set('json spaces', 4);
+
+    app.use((req, res, next) => {
+        req.logger = logger;
+        next();
+    });
+
     app.get('/status', (req, res) => {
-        res.send({ available: running });
+        const requestHandler = handleRequest(req, res);
+        requestHandler(() => ({ available: running }));
     });
 
     app.post('/assets', (req, res) => {
@@ -44,18 +52,17 @@ module.exports = function module(context) {
         });
     });
 
-    app.delete('/assets/:asset_id', (req, res) => {
-        const assetId = req.params.asset_id;
-        const handleApiError = handleError(res, logger, 500, `Could not delete asset ${assetId}`);
+    app.delete('/assets/:assetId', (req, res) => {
+        const { assetId } = req.params;
+        const requestHandler = handleRequest(req, res, `Could not delete asset ${assetId}`);
 
         if (assetId.length !== 40) {
             res.status(400).json({ error: `asset ${assetId} is not formatted correctly, please provide the full asset_id` });
         } else {
-            assetsStore.remove(assetId)
-                .then(() => {
-                    res.status(200).json({ assetId });
-                })
-                .catch(handleApiError);
+            requestHandler(async () => {
+                await assetsStore.remove(assetId);
+                return { assetId };
+            });
         }
     });
 
@@ -76,23 +83,22 @@ module.exports = function module(context) {
 
     app.get('/assets', (req, res) => {
         const query = 'id:*';
-        assetsSearch(query, req, res)
-            .then(results => res.status(200).send(JSON.stringify(results, null, 4)));
+        assetsSearch(query, req, res);
     });
 
     app.get('/assets/:name', (req, res) => {
         const query = `id:* AND name:${req.params.name}`;
-        assetsSearch(query, req, res)
-            .then(results => res.status(200).send(JSON.stringify(results, null, 4)));
+        assetsSearch(query, req, res);
     });
 
     app.get('/assets/:name/:version', (req, res) => {
         const query = `id:* AND name:${req.params.name} AND version:${req.params.version}`;
-        assetsSearch(query, req, res)
-            .then(results => res.status(200).send(JSON.stringify(results, null, 4)));
+        assetsSearch(query, req, res);
     });
 
     function createAssetTable(query, req, res) {
+        const { size, from, sort } = getSearchOptions(req, '_created:desc');
+
         const defaults = [
             'name',
             'version',
@@ -110,26 +116,34 @@ module.exports = function module(context) {
             };
         }
 
-        assetsSearch(query, res, res)
-            .then((queryResults) => {
-                const tableStr = makeTable(req, defaults, queryResults, mapping);
-                res.status(200).send(tableStr);
+        const requestHandler = handleRequest(req, res, 'Could not get assets');
+
+        requestHandler(async () => {
+            const results = await assetsStore.search(query, from, size, sort, defaults);
+            const data = results.hits.hits;
+            const assets = _.map(data, (asset) => {
+                const record = asset._source;
+                record.id = asset._id;
+                return record;
             });
+            return makeTable(req, defaults, assets, mapping);
+        });
     }
 
     function assetsSearch(query, req, res) {
-        const handleApiError = handleError(res, logger, 500, 'Could not get assets');
-
-        return assetsStore.search(query, null, 10000, '_created:desc', ['_created', 'name', 'version', 'description'])
-            .then((results) => {
-                const data = results.hits.hits;
-                return _.map(data, (asset) => {
-                    const record = asset._source;
-                    record.id = asset._id;
-                    return record;
-                });
-            })
-            .catch(handleApiError);
+        const { size, from, sort } = getSearchOptions(req, '_created:desc');
+        const requestHandler = handleRequest(req, res, 'Could not get assets');
+
+        requestHandler(async () => {
+            const fields = ['_created', 'name', 'version', 'description'];
+            const results = await assetsStore.search(query, from, size, sort, fields);
+            const data = results.hits.hits;
+            return _.map(data, (asset) => {
+                const record = asset._source;
+                record.id = asset._id;
+                return record;
+            });
+        });
     }
 
 

diff --git a/packages/teraslice/lib/processors/stdout/processor.js b/packages/teraslice/lib/processors/stdout/processor.js
@@ -1,13 +1,16 @@
 'use strict';
 
+/* eslint-disable no-console */
+
+const _ = require('lodash');
 const { BatchProcessor } = require('@terascope/job-components');
 
 class Stdout extends BatchProcessor {
     async onBatch(data) {
         if (this.opConfig.limit === 0) {
-            console.log(data); // eslint-disable-line
+            console.log(data);
         } else {
-            console.log(_.take(data, opConfig.limit)); // eslint-disable-line
+            console.log(_.take(data, this.opConfig.limit));
         }
         return data;
     }

diff --git a/packages/teraslice/lib/utils/api_utils.js b/packages/teraslice/lib/utils/api_utils.js
@@ -7,13 +7,16 @@ const parseError = require('@terascope/error-parser');
 
 function makeTable(req, defaults, data, mappingFn) {
     const query = fieldsQuery(req.query, defaults);
+    let emptyChar = 'N/A';
+
     // used to create an empty table if there are no jobs
     if (data.length === 0) {
+        emptyChar = '';
         data.push({});
     }
 
     return Table.print(data, (item, cell) => {
-        const fn = mappingFn ? mappingFn(item) : field => item[field] || 'N/A';
+        const fn = mappingFn ? mappingFn(item) : field => item[field] || emptyChar;
         _.each(query, (field) => {
             cell(field, fn(field));
         });
@@ -39,17 +42,28 @@ function fieldsQuery(query, defaults) {
     return results;
 }
 
-function handleError(res, logger, defualtCode, defaultErrorMsg) {
-    return (errObj) => {
-        if (_.isError(errObj) || errObj.code || errObj.statusCode) {
-            const code = errObj.statusCode || errObj.code || 500;
-            logger.error(errObj.message);
-            sendError(res, code, errObj.message);
-            return;
+function handleRequest(req, res, defaultErrorMsg = 'Failure to process request', { errorCode = 500, successCode = 200 } = {}) {
+    logRequest(req);
+    return async (fn) => {
+        try {
+            const result = await fn();
+            if (_.isString(result)) {
+                res.status(successCode).send(result);
+            } else {
+                res.status(successCode).json(result);
+            }
+        } catch (err) {
+            if (_.isError(err) || err.code || err.statusCode) {
+                const code = err.statusCode || err.code || errorCode;
+                req.logger.error(err.message);
+                sendError(res, code, err.message);
+                return;
+            }
+
+            const errMsg = `${defaultErrorMsg}, error: ${parseError(err)}`;
+            req.logger.error(errMsg);
+            sendError(res, errorCode, errMsg);
         }
-        const errMsg = `${defaultErrorMsg}, error: ${parseError(errObj)}`;
-        logger.error(errMsg);
-        sendError(res, defualtCode, errMsg);
     };
 }
 
@@ -103,16 +117,23 @@ function isPrometheusRequest(req) {
     return acceptHeader && acceptHeader.indexOf('application/openmetrics-text;') > -1;
 }
 
-function getSearchOptions(req) {
-    const { size = 1000, from, sort = '_updated:asc' } = req.query;
+function getSearchOptions(req, defaultSort = '_updated:asc') {
+    const { size = 100, from = null, sort = defaultSort } = req.query;
     return { size, from, sort };
 }
 
+function logRequest(req) {
+    const queryInfo = _.map(req.query, (val, key) => `${key}: ${val}`).join(', ');
+    const { method, path } = req;
+    req.logger.trace(`${_.toUpper(method)} ${path} endpoint has been called, ${queryInfo}`);
+}
+
 module.exports = {
     isPrometheusRequest,
     makePrometheus,
     makeTable,
+    logRequest,
     getSearchOptions,
-    handleError,
+    handleRequest,
     sendError
 };
diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
@@ -74,6 +74,7 @@
         "eslint": "^5.8.0",
         "eslint-config-airbnb-base": "^13.1.0",
         "eslint-plugin-import": "^2.14.0",
+        "got": "^9.3.1",
         "jest": "^23.6.0",
         "jest-extended": "^0.11.0",
         "jest-fixtures": "^0.6.0",

diff --git a/packages/teraslice/test/services/api-spec.js b/packages/teraslice/test/services/api-spec.js
@@ -0,0 +1,81 @@
+'use strict';
+
+const got = require('got');
+const express = require('express');
+const { TestContext } = require('@terascope/job-components');
+const { version } = require('../../package.json');
+const { findPort } = require('../../lib/utils/port_utils');
+const makeAPI = require('../../lib/cluster/services/api');
+
+describe('HTTP API', () => {
+    const app = express();
+    const assetsUrl = 'http://example.asset:1234';
+    const context = new TestContext('http-api');
+    context.services = {
+        execution: {
+
+        },
+        jobs: {
+
+        }
+    };
+
+    const stateStore = {
+
+    };
+
+    let api;
+    let port;
+    let baseUrl;
+    let server;
+
+    beforeAll(async () => {
+        port = await findPort();
+
+        baseUrl = `http://localhost:${port}`;
+
+        api = await makeAPI(context, app, { assetsUrl, stateStore });
+
+        await new Promise((resolve, reject) => {
+            server = app.listen(port, (err) => {
+                if (err) reject(err);
+                else resolve();
+            });
+        });
+    });
+
+    afterAll(async () => {
+        if (api) {
+            await api.shutdown();
+        }
+        if (server) {
+            await new Promise((resolve, reject) => {
+                server.close((err) => {
+                    if (err) reject(err);
+                    else resolve();
+                });
+            });
+        }
+    });
+
+    describe('GET /', () => {
+        it('should the correct response', async () => {
+            let response;
+
+            try {
+                response = await got('/', { baseUrl, json: true });
+            } catch (err) {
+                expect(err.stack).toBeNil();
+            }
+
+            expect(response.body).toMatchObject({
+                arch: context.arch,
+                clustering_type: context.sysconfig.teraslice.cluster_manager_type,
+                name: context.sysconfig.teraslice.name,
+                node_version: process.version,
+                platform: context.platform,
+                teraslice_version: `v${version}`
+            });
+        });
+    });
+});