feat(apis): init worker auto ✨

lib/helpers/montaineRequest.js

@@ -7,6 +7,7 @@ const jwt = require('jsonwebtoken');
 const path = require('path');
 const _ = require('lodash');
 
+const model = require(path.resolve('./lib/middlewares/model'));
 const config = require(path.resolve('./config'));
 const AppError = require(path.resolve('./lib/helpers/AppError'));
 
@@ -83,3 +84,27 @@ exports.setApiHistory = (status, err, start) => ({
   err: err || null,
   time: new Date() - start,
 });
+
+/**
+ * @desc Check model is Valid with Joi schema
+ * @param {Object} data - scraping result
+ * @param {Object} data - scraping result
+ * @return {Object} result
+ */
+exports.isValidDynamicSchema = (schema, obj) => {
+  const options = _.clone(config.joi.validationOptions);
+  const result = model.getResultFromJoi(obj, schema, options);
+  // if error
+  if (result && result.error) {
+    if (result.error.original && (result.error.original.password || result.error.original.firstname)) result.error.original = _.pick(result.error.original, config.whitelists.users.default);
+    let desription = '';
+    result.error.details.forEach((err) => {
+      desription += (`${err.message.charAt(0).toUpperCase() + err.message.slice(1).toLowerCase()}. `);
+    });
+    return {
+      desription,
+      error: result.error,
+    };
+  }
+  return result;
+};

modules/apis/repositories/apis.repository.js

@@ -133,7 +133,7 @@ exports.getApi = (collection, filters) => {
  * @param {Object} locations
  * @return {Object} locations
  */
-exports.getAggregateApi = (collection) => {
+exports.getAggregateApi = (collection, request) => {
   const _schema = new mongoose.Schema({}, {
     collection,
     strict: false,
@@ -146,69 +146,5 @@ exports.getAggregateApi = (collection) => {
   } catch (error) {
     model = mongoose.model(collection, _schema);
   }
-  return model.aggregate([
-    {
-      $match: {
-        $gte: '2020-04-16T00:00:01+02:00',
-        $lt: '2020-04-17T23:59:59+02:00',
-      },
-    },
-    {
-      $unwind: { path: '$weathers' },
-    },
-    {
-      $project: {
-        status: { $arrayElemAt: ['$weathers.status.value', 0] },
-        temp: { $arrayElemAt: ['$weathers.temp.value', 0] },
-      },
-    },
-    {
-      $group: {
-        _id: '$_id',
-        status: { $avg: '$status' },
-        temp: { $avg: '$temp' },
-      },
-    },
-  ]).sort('-updatedAt').exec();
+  return model.aggregate(request).exec();
 };
-
-
-// db.meteoFrance.aggregate([
-//   {
-//     $match: { '@date': '2020-03-26T00:00:00+01:00' },
-//   },
-//   {
-//     $unwind: { path: '$weathers' },
-//   },
-//   {
-//     $project: {
-//       status: { $arrayElemAt: ['$weathers.status.value', 0] },
-//       temp: { $arrayElemAt: ['$weathers.temp.value', 0] },
-//     },
-//   },
-//   {
-//     $group: {
-//       _id: '$_id',
-//       status: { $avg: '$status' },
-//       temp: { $avg: '$temp' },
-//     },
-//   },
-// ]);
-// db.mareeInfo.aggregate([
-//   {
-//     $match: { '@date': '2020-04-04T00:00:00+02:00' },
-//   },
-//   {
-//     $unwind: { path: '$coeffs' },
-//   },
-//   {
-//     $project: {
-//       coeff: { $arrayElemAt: ['$coeffs.coeff.value', 0] },
-//       height: { $arrayElemAt: ['$coeffs.height.value', 0] },
-//       hour: { $arrayElemAt: ['$coeffs.hour.value', 0] },
-//     },
-//   },
-//   {
-//     $match: { coeff: { $gt: 60 } },
-//   },
-// ]);

modules/apis/services/apis.service.js

@@ -3,7 +3,9 @@
  */
 const path = require('path');
 const _ = require('lodash');
+const Joi = require('joi');
 
+const AppError = require(path.resolve('./lib/helpers/AppError'));
 const UserService = require(path.resolve('./modules/users/services/user.service.js'));
 const montaineMap = require(path.resolve('./lib/helpers/montaineMap'));
 const montaineType = require(path.resolve('./lib/helpers/montaineType'));
@@ -135,6 +137,56 @@ exports.load = async (api, start) => {
   });
 };
 
+/**
+ * @desc Functio to ask repository to load an api request
+ * @param {Object} scrap - original scrap
+ * @return {Promise} scrap
+ */
+exports.workerAuto = async (api, body, start) => {
+  const result = {};
+
+  // generate params
+  const params = {};
+  body.forEach((el) => {
+    if (el.$match) {
+      Object.keys(api.params).forEach((key) => {
+        params[key] = el.$match[key];
+      });
+    }
+  });
+
+  // dynamic schema test on params
+  const schema = {};
+  Object.keys(api.params).forEach((key) => {
+    schema[key] = Joi.string().trim().required();
+  });
+  const paramsSchema = Joi.object().keys(schema);
+  const testedSchema = montaineRequest.isValidDynamicSchema(paramsSchema, params);
+  if (testedSchema.error) throw new AppError('Schema validation error', { code: 'SERVICE_ERROR', details: testedSchema.error });
+
+
+  // request
+  const request = await montaineRequest.request(api, params);
+  result.result = request.data.result;
+
+  // Mapping
+  if (result.result && api.mapping && api.mapping !== '') {
+    result.result = montaineMap.map(result.result, JSON.parse(api.mapping));
+  }
+
+  // Typing
+  if (result.result && api.typing && api.typing !== '') {
+    result.result = montaineType.type(result.result, JSON.parse(api.typing));
+  }
+
+  // prepare for save
+  if (result.result) {
+    result.result = montaineSave.prepare(result.result, start);
+    result.result = montaineSave.save(result.result, start);
+    if (api.savedb) result.result = await ApisRepository.import(api.slug, _.cloneDeep(result.result));
+  }
+};
+
 
 /**
  * @desc Functio to ask repository to get data stocker from apis request
@@ -163,5 +215,11 @@ exports.getApi = async (api, body) => {
  */
 exports.getAggregateApi = async (api, body) => {
   const result = await ApisRepository.getAggregateApi(api.slug, body);
+  // check if no data return, then we probably have no data :)
+  // ask for it !
+  if (result.length === 0) {
+    this.workerAuto(api, body, new Date());
+    console.log('worker auto start', body);
+  }
   return Promise.resolve(result);
 };