diff --git a/lib/helpers/montaineRequest.js b/lib/helpers/montaineRequest.js index a030861e3..59c9c326c 100644 --- a/lib/helpers/montaineRequest.js +++ b/lib/helpers/montaineRequest.js @@ -7,6 +7,7 @@ const jwt = require('jsonwebtoken'); const path = require('path'); const _ = require('lodash'); +const model = require(path.resolve('./lib/middlewares/model')); const config = require(path.resolve('./config')); const AppError = require(path.resolve('./lib/helpers/AppError')); @@ -83,3 +84,27 @@ exports.setApiHistory = (status, err, start) => ({ err: err || null, time: new Date() - start, }); + +/** + * @desc Check model is Valid with Joi schema + * @param {Object} data - scraping result + * @param {Object} data - scraping result + * @return {Object} result + */ +exports.isValidDynamicSchema = (schema, obj) => { + const options = _.clone(config.joi.validationOptions); + const result = model.getResultFromJoi(obj, schema, options); + // if error + if (result && result.error) { + if (result.error.original && (result.error.original.password || result.error.original.firstname)) result.error.original = _.pick(result.error.original, config.whitelists.users.default); + let desription = ''; + result.error.details.forEach((err) => { + desription += (`${err.message.charAt(0).toUpperCase() + err.message.slice(1).toLowerCase()}. `); + }); + return { + desription, + error: result.error, + }; + } + return result; +}; diff --git a/modules/apis/repositories/apis.repository.js b/modules/apis/repositories/apis.repository.js index df094a8b1..3fcba3263 100644 --- a/modules/apis/repositories/apis.repository.js +++ b/modules/apis/repositories/apis.repository.js @@ -133,7 +133,7 @@ exports.getApi = (collection, filters) => { * @param {Object} locations * @return {Object} locations */ -exports.getAggregateApi = (collection) => { +exports.getAggregateApi = (collection, request) => { const _schema = new mongoose.Schema({}, { collection, strict: false, @@ -146,69 +146,5 @@ exports.getAggregateApi = (collection) => { } catch (error) { model = mongoose.model(collection, _schema); } - return model.aggregate([ - { - $match: { - $gte: '2020-04-16T00:00:01+02:00', - $lt: '2020-04-17T23:59:59+02:00', - }, - }, - { - $unwind: { path: '$weathers' }, - }, - { - $project: { - status: { $arrayElemAt: ['$weathers.status.value', 0] }, - temp: { $arrayElemAt: ['$weathers.temp.value', 0] }, - }, - }, - { - $group: { - _id: '$_id', - status: { $avg: '$status' }, - temp: { $avg: '$temp' }, - }, - }, - ]).sort('-updatedAt').exec(); + return model.aggregate(request).exec(); }; - - -// db.meteoFrance.aggregate([ -// { -// $match: { '@date': '2020-03-26T00:00:00+01:00' }, -// }, -// { -// $unwind: { path: '$weathers' }, -// }, -// { -// $project: { -// status: { $arrayElemAt: ['$weathers.status.value', 0] }, -// temp: { $arrayElemAt: ['$weathers.temp.value', 0] }, -// }, -// }, -// { -// $group: { -// _id: '$_id', -// status: { $avg: '$status' }, -// temp: { $avg: '$temp' }, -// }, -// }, -// ]); -// db.mareeInfo.aggregate([ -// { -// $match: { '@date': '2020-04-04T00:00:00+02:00' }, -// }, -// { -// $unwind: { path: '$coeffs' }, -// }, -// { -// $project: { -// coeff: { $arrayElemAt: ['$coeffs.coeff.value', 0] }, -// height: { $arrayElemAt: ['$coeffs.height.value', 0] }, -// hour: { $arrayElemAt: ['$coeffs.hour.value', 0] }, -// }, -// }, -// { -// $match: { coeff: { $gt: 60 } }, -// }, -// ]); diff --git a/modules/apis/services/apis.service.js b/modules/apis/services/apis.service.js index 6fe4bfbd9..e591bd6bf 100644 --- a/modules/apis/services/apis.service.js +++ b/modules/apis/services/apis.service.js @@ -3,7 +3,9 @@ */ const path = require('path'); const _ = require('lodash'); +const Joi = require('joi'); +const AppError = require(path.resolve('./lib/helpers/AppError')); const UserService = require(path.resolve('./modules/users/services/user.service.js')); const montaineMap = require(path.resolve('./lib/helpers/montaineMap')); const montaineType = require(path.resolve('./lib/helpers/montaineType')); @@ -135,6 +137,56 @@ exports.load = async (api, start) => { }); }; +/** + * @desc Functio to ask repository to load an api request + * @param {Object} scrap - original scrap + * @return {Promise} scrap + */ +exports.workerAuto = async (api, body, start) => { + const result = {}; + + // generate params + const params = {}; + body.forEach((el) => { + if (el.$match) { + Object.keys(api.params).forEach((key) => { + params[key] = el.$match[key]; + }); + } + }); + + // dynamic schema test on params + const schema = {}; + Object.keys(api.params).forEach((key) => { + schema[key] = Joi.string().trim().required(); + }); + const paramsSchema = Joi.object().keys(schema); + const testedSchema = montaineRequest.isValidDynamicSchema(paramsSchema, params); + if (testedSchema.error) throw new AppError('Schema validation error', { code: 'SERVICE_ERROR', details: testedSchema.error }); + + + // request + const request = await montaineRequest.request(api, params); + result.result = request.data.result; + + // Mapping + if (result.result && api.mapping && api.mapping !== '') { + result.result = montaineMap.map(result.result, JSON.parse(api.mapping)); + } + + // Typing + if (result.result && api.typing && api.typing !== '') { + result.result = montaineType.type(result.result, JSON.parse(api.typing)); + } + + // prepare for save + if (result.result) { + result.result = montaineSave.prepare(result.result, start); + result.result = montaineSave.save(result.result, start); + if (api.savedb) result.result = await ApisRepository.import(api.slug, _.cloneDeep(result.result)); + } +}; + /** * @desc Functio to ask repository to get data stocker from apis request @@ -163,5 +215,11 @@ exports.getApi = async (api, body) => { */ exports.getAggregateApi = async (api, body) => { const result = await ApisRepository.getAggregateApi(api.slug, body); + // check if no data return, then we probably have no data :) + // ask for it ! + if (result.length === 0) { + this.workerAuto(api, body, new Date()); + console.log('worker auto start', body); + } return Promise.resolve(result); };