diff --git a/lib/api3/generic/collection.js b/lib/api3/generic/collection.js index 0a1a29b3915d..015af2a13f42 100644 --- a/lib/api3/generic/collection.js +++ b/lib/api3/generic/collection.js @@ -5,7 +5,8 @@ const apiConst = require('../const.json') , dateTools = require('../shared/dateTools') , opTools = require('../shared/operationTools') , stringTools = require('../shared/stringTools') - , CollectionStorage = require('../storage/mongoCollection') + , MongoCollectionStorage = require('../storage/mongoCollection') + , CachedCollectionStorage = require('../storage/mongoCachedCollection') , searchOperation = require('./search/operation') , createOperation = require('./create/operation') , readOperation = require('./read/operation') @@ -26,13 +27,16 @@ function Collection ({ ctx, env, app, colName, storageColName, fallbackGetDate, fallbackDateField }) { const self = this; - + self.colName = colName; self.fallbackGetDate = fallbackGetDate; self.dedupFallbackFields = app.get('API3_DEDUP_FALLBACK_ENABLED') ? dedupFallbackFields : []; self.autoPruneDays = app.setENVTruthy('API3_AUTOPRUNE_' + colName.toUpperCase()); self.nextAutoPrune = new Date(); - self.storage = new CollectionStorage(ctx, env, storageColName); + + const baseStorage = new MongoCollectionStorage(ctx, env, storageColName); + self.storage = new CachedCollectionStorage(ctx, env, colName, baseStorage); + self.fallbackDateField = fallbackDateField; self.mapRoutes = function mapRoutes () { @@ -89,9 +93,9 @@ function Collection ({ ctx, env, app, colName, storageColName, fallbackGetDate, }; - + /** - * Fetch modified date from document (with possible fallback and back-fill to srvModified/srvCreated) + * Fetch modified date from document (with possible fallback and back-fill to srvModified/srvCreated) * @param {Object} doc - document loaded from database */ self.resolveDates = function resolveDates (doc) { @@ -125,12 +129,12 @@ function Collection ({ ctx, env, app, colName, storageColName, fallbackGetDate, * in the background (asynchronously) * */ self.autoPrune = function autoPrune () { - + if (!stringTools.isNumberInString(self.autoPruneDays)) return; - + const autoPruneDays = parseFloat(self.autoPruneDays); - if (autoPruneDays <= 0) + if (autoPruneDays <= 0) return; if (new Date() > self.nextAutoPrune) { @@ -190,4 +194,4 @@ function Collection ({ ctx, env, app, colName, storageColName, fallbackGetDate, } } -module.exports = Collection; \ No newline at end of file +module.exports = Collection; diff --git a/lib/api3/generic/history/operation.js b/lib/api3/generic/history/operation.js index 5151c8b2749c..fb3ed0184ec0 100644 --- a/lib/api3/generic/history/operation.js +++ b/lib/api3/generic/history/operation.js @@ -27,13 +27,13 @@ async function history (opCtx, fieldsProjector) { if (filter !== null && limit !== null && projection !== null) { - const result = await col.storage.findMany(filter + const result = await col.storage.findMany({ filter , sort , limit , skip , projection , onlyValid - , logicalOperator); + , logicalOperator }); if (!result) throw new Error('empty result'); @@ -150,4 +150,4 @@ function historyOperation (ctx, env, app, col) { }; } -module.exports = historyOperation; \ No newline at end of file +module.exports = historyOperation; diff --git a/lib/api3/generic/search/operation.js b/lib/api3/generic/search/operation.js index c24f978dc277..179f357b5039 100644 --- a/lib/api3/generic/search/operation.js +++ b/lib/api3/generic/search/operation.js @@ -35,13 +35,12 @@ async function search (opCtx) { if (filter !== null && sort !== null && limit !== null && skip !== null && projection !== null) { - - const result = await col.storage.findMany(filter + const result = await col.storage.findMany({ filter , sort , limit , skip , projection - , onlyValid); + , onlyValid }); if (!result) throw new Error('empty result'); @@ -76,4 +75,4 @@ function searchOperation (ctx, env, app, col) { }; } -module.exports = searchOperation; \ No newline at end of file +module.exports = searchOperation; diff --git a/lib/api3/storage/mongoCachedCollection/index.js b/lib/api3/storage/mongoCachedCollection/index.js new file mode 100644 index 000000000000..b2fd8f21a550 --- /dev/null +++ b/lib/api3/storage/mongoCachedCollection/index.js @@ -0,0 +1,147 @@ +'use strict'; + +const _ = require('lodash') + +/** + * Storage implementation which wraps mongo baseStorage with caching + * @param {Object} ctx + * @param {Object} env + * @param {string} colName - name of the collection in mongo database + * @param {Object} baseStorage - wrapped mongo storage implementation + */ +function MongoCachedCollection (ctx, env, colName, baseStorage) { + + const self = this; + + self.colName = colName; + + self.identifyingFilter = baseStorage.identifyingFilter; + + self.findOne = (...args) => baseStorage.findOne(...args); + + self.findOneFilter = (...args) => baseStorage.findOneFilter(...args); + + self.findMany = (...args) => baseStorage.findMany(...args); + + + self.insertOne = async (doc) => { + const result = await baseStorage.insertOne(doc, { normalize: false }); + + if (cacheSupported()) { + updateInCache([doc]); + } + + if (doc._id) { + delete doc._id; + } + return result; + } + + + self.replaceOne = async (identifier, doc) => { + const result = await baseStorage.replaceOne(identifier, doc); + + if (cacheSupported()) { + const rawDocs = await baseStorage.findOne(identifier, null, { normalize: false }) + updateInCache([rawDocs[0]]) + } + + return result; + } + + + self.updateOne = async (identifier, setFields) => { + const result = await baseStorage.updateOne(identifier, setFields); + + if (cacheSupported()) { + const rawDocs = await baseStorage.findOne(identifier, null, { normalize: false }) + + if (rawDocs[0].isValid === false) { + deleteInCache(rawDocs) + } + else { + updateInCache([rawDocs[0]]) + } + } + + return result; + } + + self.deleteOne = async (identifier) => { + let invalidateDocs + if (cacheSupported()) { + invalidateDocs = await baseStorage.findOne(identifier, { _id: 1 }, { normalize: false }) + } + + const result = await baseStorage.deleteOne(identifier); + + if (cacheSupported()) { + deleteInCache(invalidateDocs) + } + + return result; + } + + self.deleteManyOr = async (filter) => { + let invalidateDocs + if (cacheSupported()) { + invalidateDocs = await baseStorage.findMany({ filter, + limit: 1000, + skip: 0, + projection: { _id: 1 }, + options: { normalize: false } }); + } + + const result = await baseStorage.deleteManyOr(filter); + + if (cacheSupported()) { + deleteInCache(invalidateDocs) + } + + return result; + } + + self.version = (...args) => baseStorage.version(...args); + + self.getLastModified = (...args) => baseStorage.getLastModified(...args); + + function cacheSupported () { + return ctx.cache + && ctx.cache[colName] + && _.isArray(ctx.cache[colName]); + } + + function updateInCache (doc) { + if (doc && doc.isValid === false) { + deleteInCache([doc._id]) + } + else { + ctx.bus.emit('data-update', { + type: colName + , op: 'update' + , changes: doc + }); + } + } + + function deleteInCache (docs) { + let changes + if (_.isArray(docs)) { + if (docs.length === 0) { + return + } + else if (docs.length === 1 && docs[0]._id) { + const _id = docs[0]._id.toString() + changes = [ _id ] + } + } + + ctx.bus.emit('data-update', { + type: colName + , op: 'remove' + , changes + }); + } +} + +module.exports = MongoCachedCollection; diff --git a/lib/api3/storage/mongoCollection/find.js b/lib/api3/storage/mongoCollection/find.js index bc399dbce98e..013d008ec98d 100644 --- a/lib/api3/storage/mongoCollection/find.js +++ b/lib/api3/storage/mongoCollection/find.js @@ -10,8 +10,9 @@ const utils = require('./utils') * @param {Object} col * @param {string} identifier * @param {Object} projection + * @param {Object} options */ -function findOne (col, identifier, projection) { +function findOne (col, identifier, projection, options) { return new Promise(function (resolve, reject) { @@ -25,7 +26,9 @@ function findOne (col, identifier, projection) { if (err) { reject(err); } else { - _.each(result, utils.normalizeDoc); + if (!options || options.normalize !== false) { + _.each(result, utils.normalizeDoc); + } resolve(result); } }); @@ -38,8 +41,9 @@ function findOne (col, identifier, projection) { * @param {Object} col * @param {Object} filter specific filter * @param {Object} projection + * @param {Object} options */ -function findOneFilter (col, filter, projection) { +function findOneFilter (col, filter, projection, options) { return new Promise(function (resolve, reject) { @@ -51,7 +55,9 @@ function findOneFilter (col, filter, projection) { if (err) { reject(err); } else { - _.each(result, utils.normalizeDoc); + if (!options || options.normalize !== false) { + _.each(result, utils.normalizeDoc); + } resolve(result); } }); @@ -62,23 +68,25 @@ function findOneFilter (col, filter, projection) { /** * Find many documents matching the filtering criteria */ -function findMany (col, filterDef, sort, limit, skip, projection, onlyValid, logicalOperator = 'and') { - +function findMany (col, args) { + const logicalOperator = args.logicalOperator || 'and'; return new Promise(function (resolve, reject) { - const filter = utils.parseFilter(filterDef, logicalOperator, onlyValid); + const filter = utils.parseFilter(args.filter, logicalOperator, args.onlyValid); col.find(filter) - .sort(sort) - .limit(limit) - .skip(skip) - .project(projection) + .sort(args.sort) + .limit(args.limit) + .skip(args.skip) + .project(args.projection) .toArray(function mongoDone (err, result) { if (err) { reject(err); } else { - _.each(result, utils.normalizeDoc); + if (!args.options || args.options.normalize !== false) { + _.each(result, utils.normalizeDoc); + } resolve(result); } }); @@ -90,4 +98,4 @@ module.exports = { findOne, findOneFilter, findMany -}; \ No newline at end of file +}; diff --git a/lib/api3/storage/mongoCollection/index.js b/lib/api3/storage/mongoCollection/index.js index e6ad0a6cf8bf..ef041ce9c1f0 100644 --- a/lib/api3/storage/mongoCollection/index.js +++ b/lib/api3/storage/mongoCollection/index.js @@ -18,7 +18,7 @@ function MongoCollection (ctx, env, colName) { self.col = ctx.store.collection(colName); - ctx.store.ensureIndexes(self.col, [ 'identifier', + ctx.store.ensureIndexes(self.col, [ 'identifier', 'srvModified', 'isValid' ]); @@ -64,10 +64,10 @@ function MongoCollection (ctx, env, colName) { /** - * Get timestamp (e.g. srvModified) of the last modified document + * Get timestamp (e.g. srvModified) of the last modified document */ self.getLastModified = function getLastModified (fieldName) { - + return new Promise(function (resolve, reject) { self.col.find() @@ -87,4 +87,4 @@ function MongoCollection (ctx, env, colName) { } } -module.exports = MongoCollection; \ No newline at end of file +module.exports = MongoCollection; diff --git a/lib/api3/storage/mongoCollection/modify.js b/lib/api3/storage/mongoCollection/modify.js index 6552fe40e8cc..7183f1c971a6 100644 --- a/lib/api3/storage/mongoCollection/modify.js +++ b/lib/api3/storage/mongoCollection/modify.js @@ -1,14 +1,15 @@ 'use strict'; -const utils = require('./utils'); - +const utils = require('./utils') + ; /** * Insert single document * @param {Object} col * @param {Object} doc + * @param {Object} options */ -function insertOne (col, doc) { +function insertOne (col, doc, options) { return new Promise(function (resolve, reject) { @@ -18,7 +19,10 @@ function insertOne (col, doc) { reject(err); } else { const identifier = doc.identifier || result.insertedId.toString(); - delete doc._id; + + if (!options || options.normalize !== false) { + delete doc._id; + } resolve(identifier); } }); @@ -38,7 +42,7 @@ function replaceOne (col, identifier, doc) { const filter = utils.filterForOne(identifier); - col.replaceOne(filter, doc, { }, function mongoDone(err, result) { + col.replaceOne(filter, doc, { upsert: true }, function mongoDone(err, result) { if (err) { reject(err); } else { @@ -120,4 +124,4 @@ module.exports = { updateOne, deleteOne, deleteManyOr -}; \ No newline at end of file +}; diff --git a/lib/api3/storage/mongoCollection/utils.js b/lib/api3/storage/mongoCollection/utils.js index 1b2ab5610d7c..a2f7b16520c5 100644 --- a/lib/api3/storage/mongoCollection/utils.js +++ b/lib/api3/storage/mongoCollection/utils.js @@ -26,7 +26,6 @@ function normalizeDoc (doc) { * @param {bool} onlyValid */ function parseFilter (filterDef, logicalOperator, onlyValid) { - let filter = { }; if (!filterDef) return filter; @@ -175,4 +174,4 @@ module.exports = { parseFilter, filterForOne, identifyingFilter -}; \ No newline at end of file +}; diff --git a/lib/server/cache.js b/lib/server/cache.js index 88e5f77ca02f..5b92e42f58dc 100644 --- a/lib/server/cache.js +++ b/lib/server/cache.js @@ -7,7 +7,7 @@ * to be accessed by the persistence layer as data is inserted, updated * or deleted, as well as the periodic dataloader, which polls Mongo * for new inserts. - * + * * Longer term, the cache is planned to allow skipping the Mongo polls * altogether. */ diff --git a/tests/api3.create.test.js b/tests/api3.create.test.js index 22c88d3d16d8..d6e1406bec1a 100644 --- a/tests/api3.create.test.js +++ b/tests/api3.create.test.js @@ -61,13 +61,15 @@ describe('API3 CREATE', function() { self.app = self.instance.app; self.env = self.instance.env; - self.url = '/api/v3/treatments'; + self.col = 'treatments' + self.url = `/api/v3/${self.col}`; let authResult = await authSubject(self.instance.ctx.authorization.storage); self.subject = authResult.subject; self.token = authResult.token; self.urlToken = `${self.url}?token=${self.token.create}`; + self.cache = self.instance.cacheMonitor; }); @@ -76,6 +78,16 @@ describe('API3 CREATE', function() { }); + beforeEach(() => { + self.cache.clear(); + }); + + + afterEach(() => { + self.cache.shouldBeEmpty(); + }); + + it('should require authentication', async () => { let res = await self.instance.post(`${self.url}`) .send(self.validDoc) @@ -123,6 +135,7 @@ describe('API3 CREATE', function() { let body = await self.get(self.validDoc.identifier); body.should.containEql(self.validDoc); + self.cache.nextShouldEql(self.col, self.validDoc) const ms = body.srvModified % 1000; (body.srvModified - ms).should.equal(lastModified); @@ -130,6 +143,7 @@ describe('API3 CREATE', function() { body.subject.should.equal(self.subject.apiCreate.name); await self.delete(self.validDoc.identifier); + self.cache.nextShouldDeleteLast(self.col) }); @@ -218,13 +232,18 @@ describe('API3 CREATE', function() { it('should accept valid utcOffset', async () => { + const doc = Object.assign({}, self.validDoc, { utcOffset: 120 }); + await self.instance.post(self.urlToken) - .send(Object.assign({}, self.validDoc, { utcOffset: 120 })) + .send(doc) .expect(201); let body = await self.get(self.validDoc.identifier); body.utcOffset.should.equal(120); + self.cache.nextShouldEql(self.col, doc) + await self.delete(self.validDoc.identifier); + self.cache.nextShouldDeleteLast(self.col) }); @@ -279,7 +298,10 @@ describe('API3 CREATE', function() { let body = await self.get(self.validDoc.identifier); body.date.should.equal(1560146828576); body.utcOffset.should.equal(120); + self.cache.nextShouldEql(self.col, body) + await self.delete(self.validDoc.identifier); + self.cache.nextShouldDeleteLast(self.col) }); @@ -295,6 +317,7 @@ describe('API3 CREATE', function() { let createdBody = await self.get(doc.identifier); createdBody.should.containEql(doc); + self.cache.nextShouldEql(self.col, doc) const doc2 = Object.assign({}, doc); let res = await self.instance.post(self.urlToken) @@ -304,6 +327,7 @@ describe('API3 CREATE', function() { res.body.status.should.equal(403); res.body.message.should.equal('Missing permission api:treatments:update'); await self.delete(doc.identifier); + self.cache.nextShouldDeleteLast(self.col) }); @@ -319,6 +343,7 @@ describe('API3 CREATE', function() { let createdBody = await self.get(doc.identifier); createdBody.should.containEql(doc); + self.cache.nextShouldEql(self.col, doc) const doc2 = Object.assign({}, doc, { insulin: 0.5 @@ -330,8 +355,10 @@ describe('API3 CREATE', function() { let updatedBody = await self.get(doc2.identifier); updatedBody.should.containEql(doc2); + self.cache.nextShouldEql(self.col, doc2) await self.delete(doc2.identifier); + self.cache.nextShouldDeleteLast(self.col) }); @@ -339,29 +366,37 @@ describe('API3 CREATE', function() { self.validDoc.date = (new Date()).getTime(); self.validDoc.identifier = utils.randomString('32', 'aA#'); - const doc = Object.assign({}, self.validDoc, { - created_at: new Date(self.validDoc.date).toISOString() + const doc = Object.assign({}, self.validDoc, { + created_at: new Date(self.validDoc.date).toISOString() }); delete doc.identifier; - self.instance.ctx.treatments.create([doc], async (err) => { // let's insert the document in APIv1's way - should.not.exist(err); + await new Promise((resolve, reject) => { + self.instance.ctx.treatments.create([doc], async (err) => { // let's insert the document in APIv1's way + should.not.exist(err); + doc._id = doc._id.toString(); + self.cache.nextShouldEql(self.col, doc) - const doc2 = Object.assign({}, doc, { - insulin: 0.4, - identifier: utils.randomString('32', 'aA#') + err ? reject(err) : resolve(doc); }); - delete doc2._id; // APIv1 updates input document, we must get rid of _id for the next round + }); - await self.instance.post(`${self.url}?token=${self.token.all}`) - .send(doc2) - .expect(204); + const doc2 = Object.assign({}, doc, { + insulin: 0.4, + identifier: utils.randomString('32', 'aA#') + }); + delete doc2._id; // APIv1 updates input document, we must get rid of _id for the next round + + await self.instance.post(`${self.url}?token=${self.token.all}`) + .send(doc2) + .expect(204); - let updatedBody = await self.get(doc2.identifier); - updatedBody.should.containEql(doc2); + let updatedBody = await self.get(doc2.identifier); + updatedBody.should.containEql(doc2); + self.cache.nextShouldEql(self.col, doc2) - await self.delete(doc2.identifier); - }); + await self.delete(doc2.identifier); + self.cache.nextShouldDeleteLast(self.col) }); @@ -369,51 +404,62 @@ describe('API3 CREATE', function() { self.validDoc.date = (new Date()).getTime(); self.validDoc.identifier = utils.randomString('32', 'aA#'); - const doc = Object.assign({}, self.validDoc, { - created_at: new Date(self.validDoc.date).toISOString() + const doc = Object.assign({}, self.validDoc, { + created_at: new Date(self.validDoc.date).toISOString() }); delete doc.identifier; - let p = await new Promise(function(resolve, reject) { - self.instance.ctx.treatments.create([doc], async (err) => { // let's insert the document in APIv1's way + await new Promise((resolve, reject) => { + self.instance.ctx.treatments.create([doc], async (err) => { // let's insert the document in APIv1's way should.not.exist(err); + doc._id = doc._id.toString(); - let oldBody = await self.get(doc._id); - delete doc._id; // APIv1 updates input document, we must get rid of _id for the next round - oldBody.should.containEql(doc); - - const doc2 = Object.assign({}, doc, { - eventType: 'Meal Bolus', - insulin: 0.4, - identifier: utils.randomString('32', 'aA#') - }); + self.cache.nextShouldEql(self.col, doc) + err ? reject(err) : resolve(doc); + }); + }); - await self.instance.post(`${self.url}?token=${self.token.all}`) - .send(doc2) - .expect(201); + const oldBody = await self.get(doc._id); + delete doc._id; // APIv1 updates input document, we must get rid of _id for the next round + oldBody.should.containEql(doc); - let updatedBody = await self.get(doc2.identifier); - updatedBody.should.containEql(doc2); - updatedBody.identifier.should.not.equal(oldBody.identifier); - await self.delete(doc2.identifier); - await self.delete(oldBody.identifier); - resolve('Done!'); - }); + const doc2 = Object.assign({}, doc, { + eventType: 'Meal Bolus', + insulin: 0.4, + identifier: utils.randomString('32', 'aA#') }); + + await self.instance.post(`${self.url}?token=${self.token.all}`) + .send(doc2) + .expect(201); + + let updatedBody = await self.get(doc2.identifier); + updatedBody.should.containEql(doc2); + updatedBody.identifier.should.not.equal(oldBody.identifier); + self.cache.nextShouldEql(self.col, doc2) + + await self.delete(doc2.identifier); + self.cache.nextShouldDeleteLast(self.col) + + await self.delete(oldBody.identifier); + self.cache.nextShouldDeleteLast(self.col) }); it('should overwrite deleted document', async () => { const date1 = new Date() - , identifier = utils.randomString('32', 'aA#'); + , identifier = utils.randomString('32', 'aA#') + , doc = Object.assign({}, self.validDoc, { identifier, date: date1.toISOString() }); await self.instance.post(self.urlToken) - .send(Object.assign({}, self.validDoc, { identifier, date: date1.toISOString() })) + .send(doc) .expect(201); + self.cache.nextShouldEql(self.col, Object.assign({}, doc, { date: date1.getTime() })); await self.instance.delete(`${self.url}/${identifier}?token=${self.token.delete}`) .expect(204); + self.cache.nextShouldDeleteLast(self.col) const date2 = new Date(); let res = await self.instance.post(self.urlToken) @@ -422,17 +468,22 @@ describe('API3 CREATE', function() { res.body.status.should.be.equal(403); res.body.message.should.be.equal('Missing permission api:treatments:update'); + self.cache.shouldBeEmpty() + const doc2 = Object.assign({}, self.validDoc, { identifier, date: date2.toISOString() }); res = await self.instance.post(`${self.url}?token=${self.token.all}`) - .send(Object.assign({}, self.validDoc, { identifier, date: date2.toISOString() })) + .send(doc2) .expect(204); res.body.should.be.empty(); + self.cache.nextShouldEql(self.col, Object.assign({}, doc2, { date: date2.getTime() })); let body = await self.get(identifier); body.date.should.equal(date2.getTime()); body.identifier.should.equal(identifier); + await self.delete(identifier); + self.cache.nextShouldDeleteLast(self.col) }); @@ -451,7 +502,10 @@ describe('API3 CREATE', function() { let body = await self.get(validIdentifier); body.should.containEql(self.validDoc); + self.cache.nextShouldEql(self.col, self.validDoc); + await self.delete(validIdentifier); + self.cache.nextShouldDeleteLast(self.col) }); @@ -470,6 +524,7 @@ describe('API3 CREATE', function() { let body = await self.get(validIdentifier); body.should.containEql(self.validDoc); + self.cache.nextShouldEql(self.col, self.validDoc); delete self.validDoc.identifier; res = await self.instance.post(`${self.url}?token=${self.token.update}`) @@ -479,11 +534,13 @@ describe('API3 CREATE', function() { res.body.should.be.empty(); res.headers.location.should.equal(`${self.url}/${validIdentifier}`); self.validDoc.identifier = validIdentifier; + self.cache.nextShouldEql(self.col, self.validDoc); body = await self.search(self.validDoc.date); body.length.should.equal(1); await self.delete(validIdentifier); + self.cache.nextShouldDeleteLast(self.col) }); }); diff --git a/tests/api3.delete.test.js b/tests/api3.delete.test.js index 7cee15410a05..5ca87a0d57be 100644 --- a/tests/api3.delete.test.js +++ b/tests/api3.delete.test.js @@ -24,6 +24,7 @@ describe('API3 UPDATE', function() { self.subject = authResult.subject; self.token = authResult.token; self.urlToken = `${self.url}?token=${self.token.delete}`; + self.cache = self.instance.cacheMonitor; }); @@ -32,6 +33,16 @@ describe('API3 UPDATE', function() { }); + beforeEach(() => { + self.cache.clear(); + }); + + + afterEach(() => { + self.cache.shouldBeEmpty(); + }); + + it('should require authentication', async () => { let res = await self.instance.delete(`${self.url}/FAKE_IDENTIFIER`) .expect(401); diff --git a/tests/api3.generic.workflow.test.js b/tests/api3.generic.workflow.test.js index 7cfbc53f618f..49bd7664d30b 100644 --- a/tests/api3.generic.workflow.test.js +++ b/tests/api3.generic.workflow.test.js @@ -31,7 +31,8 @@ describe('Generic REST API3', function() { self.app = self.instance.app; self.env = self.instance.env; - self.urlCol = '/api/v3/treatments'; + self.col = 'treatments'; + self.urlCol = `/api/v3/${self.col}`; self.urlResource = self.urlCol + '/' + self.identifier; self.urlHistory = self.urlCol + '/history'; @@ -40,6 +41,7 @@ describe('Generic REST API3', function() { self.subject = authResult.subject; self.token = authResult.token; self.urlToken = `${self.url}?token=${self.token.create}`; + self.cache = self.instance.cacheMonitor; }); @@ -48,6 +50,16 @@ describe('Generic REST API3', function() { }); + beforeEach(() => { + self.cache.clear(); + }); + + + afterEach(() => { + self.cache.shouldBeEmpty(); + }); + + self.checkHistoryExistence = async function checkHistoryExistence (assertions) { let res = await self.instance.get(`${self.urlHistory}/${self.historyTimestamp}?token=${self.token.read}`) @@ -113,6 +125,8 @@ describe('Generic REST API3', function() { await self.instance.post(`${self.urlCol}?token=${self.token.create}`) .send(self.docOriginal) .expect(201); + + self.cache.nextShouldEql(self.col, self.docOriginal) }); @@ -154,14 +168,17 @@ describe('Generic REST API3', function() { .expect(204); self.docActual.subject = self.subject.apiUpdate.name; + delete self.docActual.srvModified; + + self.cache.nextShouldEql(self.col, self.docActual) }); it('document changed in HISTORY', async () => { await self.checkHistoryExistence(); - }); + }); + - it('document changed in READ', async () => { let res = await self.instance.get(`${self.urlResource}?token=${self.token.read}`) .expect(200); @@ -179,14 +196,18 @@ describe('Generic REST API3', function() { await self.instance.patch(`${self.urlResource}?token=${self.token.update}`) .send({ 'carbs': self.docActual.carbs, 'insulin': self.docActual.insulin }) .expect(204); + + delete self.docActual.srvModified; + + self.cache.nextShouldEql(self.col, self.docActual) }); it('document changed in HISTORY', async () => { await self.checkHistoryExistence(); - }); + }); + - it('document changed in READ', async () => { let res = await self.instance.get(`${self.urlResource}?token=${self.token.read}`) .expect(200); @@ -200,6 +221,8 @@ describe('Generic REST API3', function() { it('soft DELETE', async () => { await self.instance.delete(`${self.urlResource}?token=${self.token.delete}`) .expect(204); + + self.cache.nextShouldDeleteLast(self.col) }); @@ -217,19 +240,21 @@ describe('Generic REST API3', function() { res.body.should.have.length(0); }); - + it('document deleted in HISTORY', async () => { await self.checkHistoryExistence(value => { value.isValid.should.be.eql(false); }); - }); - + }); + it('permanent DELETE', async () => { await self.instance.delete(`${self.urlResource}?token=${self.token.delete}`) .query({ 'permanent': 'true' }) .expect(204); + + self.cache.nextShouldDeleteLast(self.col) }); @@ -264,6 +289,9 @@ describe('Generic REST API3', function() { delete self.docActual.srvModified; const readOnlyMessage = 'Trying to modify read-only document'; + self.cache.nextShouldEql(self.col, self.docActual) + self.cache.shouldBeEmpty() + res = await self.instance.post(`${self.urlCol}?token=${self.token.update}`) .send(Object.assign({}, self.docActual, { insulin: 0.41 })) .expect(422); diff --git a/tests/api3.patch.test.js b/tests/api3.patch.test.js index 36dccc94bfaf..750bb37b7456 100644 --- a/tests/api3.patch.test.js +++ b/tests/api3.patch.test.js @@ -20,7 +20,7 @@ describe('API3 PATCH', function() { insulin: 0.3 }; self.validDoc.identifier = opTools.calculateIdentifier(self.validDoc); - + self.timeout(15000); @@ -40,13 +40,15 @@ describe('API3 PATCH', function() { self.app = self.instance.app; self.env = self.instance.env; - self.url = '/api/v3/treatments'; + self.col = 'treatments'; + self.url = `/api/v3/${self.col}`; let authResult = await authSubject(self.instance.ctx.authorization.storage); self.subject = authResult.subject; self.token = authResult.token; self.urlToken = `${self.url}/${self.validDoc.identifier}?token=${self.token.update}`; + self.cache = self.instance.cacheMonitor; }); @@ -55,6 +57,16 @@ describe('API3 PATCH', function() { }); + beforeEach(() => { + self.cache.clear(); + }); + + + afterEach(() => { + self.cache.shouldBeEmpty(); + }); + + it('should require authentication', async () => { let res = await self.instance.patch(`${self.url}/FAKE_IDENTIFIER`) .expect(401); @@ -86,6 +98,7 @@ describe('API3 PATCH', function() { .expect(201); res.body.should.be.empty(); + self.cache.nextShouldEql(self.col, self.validDoc) }); @@ -213,6 +226,8 @@ describe('API3 PATCH', function() { body.insulin.should.equal(0.3); body.subject.should.equal(self.subject.apiCreate.name); body.modifiedBy.should.equal(self.subject.apiUpdate.name); + + self.cache.nextShouldEql(self.col, body) }); }); diff --git a/tests/api3.read.test.js b/tests/api3.read.test.js index d9f73ebf13a4..3c7f0eaf964c 100644 --- a/tests/api3.read.test.js +++ b/tests/api3.read.test.js @@ -4,13 +4,13 @@ require('should'); -describe('API3 READ', function() { +describe('API3 READ', function () { const self = this , testConst = require('./fixtures/api3/const.json') , instance = require('./fixtures/api3/instance') , authSubject = require('./fixtures/api3/authSubject') , opTools = require('../lib/api3/shared/operationTools') - ; + ; self.validDoc = { date: (new Date()).getTime(), @@ -28,12 +28,14 @@ describe('API3 READ', function() { self.app = self.instance.app; self.env = self.instance.env; - self.url = '/api/v3/devicestatus'; + self.col = 'devicestatus'; + self.url = `/api/v3/${self.col}`; let authResult = await authSubject(self.instance.ctx.authorization.storage); self.subject = authResult.subject; self.token = authResult.token; + self.cache = self.instance.cacheMonitor; }); @@ -42,6 +44,16 @@ describe('API3 READ', function() { }); + beforeEach(() => { + self.cache.clear(); + }); + + + afterEach(() => { + self.cache.shouldBeEmpty(); + }); + + it('should require authentication', async () => { let res = await self.instance.get(`${self.url}/FAKE_IDENTIFIER`) .expect(401); @@ -57,12 +69,16 @@ describe('API3 READ', function() { .expect(404); res.body.should.be.empty(); + + self.cache.shouldBeEmpty() }); it('should not found not existing document', async () => { await self.instance.get(`${self.url}/${self.validDoc.identifier}?token=${self.token.read}`) .expect(404); + + self.cache.shouldBeEmpty() }); @@ -81,6 +97,8 @@ describe('API3 READ', function() { res.body.should.have.property('srvModified').which.is.a.Number(); res.body.should.have.property('subject'); self.validDoc.subject = res.body.subject; // let's store subject for later tests + + self.cache.nextShouldEql(self.col, self.validDoc) }); @@ -130,6 +148,7 @@ describe('API3 READ', function() { .expect(204); res.body.should.be.empty(); + self.cache.nextShouldDeleteLast(self.col) res = await self.instance.get(`${self.url}/${self.validDoc.identifier}?token=${self.token.read}`) .expect(410); @@ -143,6 +162,7 @@ describe('API3 READ', function() { .expect(204); res.body.should.be.empty(); + self.cache.nextShouldDeleteLast(self.col) res = await self.instance.get(`${self.url}/${self.validDoc.identifier}?token=${self.token.read}`) .expect(404); @@ -153,28 +173,38 @@ describe('API3 READ', function() { it('should found document created by APIv1', async () => { - const doc = Object.assign({}, self.validDoc, { - created_at: new Date(self.validDoc.date).toISOString() + const doc = Object.assign({}, self.validDoc, { + created_at: new Date(self.validDoc.date).toISOString() }); delete doc.identifier; - self.instance.ctx.devicestatus.create([doc], async (err) => { // let's insert the document in APIv1's way - should.not.exist(err); - const identifier = doc._id.toString(); - delete doc._id; + await new Promise((resolve, reject) => { + self.instance.ctx.devicestatus.create([doc], async (err) => { // let's insert the document in APIv1's way - let res = await self.instance.get(`${self.url}/${identifier}?token=${self.token.read}`) - .expect(200); + should.not.exist(err); + doc._id = doc._id.toString(); + self.cache.nextShouldEql(self.col, doc) - res.body.should.containEql(doc); + err ? reject(err) : resolve(doc); + }); + }); - res = await self.instance.delete(`${self.url}/${identifier}?permanent=true&token=${self.token.delete}`) - .expect(204); + const identifier = doc._id.toString(); + delete doc._id; - res.body.should.be.empty(); - }); + let res = await self.instance.get(`${self.url}/${identifier}?token=${self.token.read}`) + .expect(200); + + res.body.should.containEql(doc); + + res = await self.instance.delete(`${self.url}/${identifier}?permanent=true&token=${self.token.delete}`) + .expect(204); + + res.body.should.be.empty(); + self.cache.nextShouldDeleteLast(self.col) }); -}); +}) +; diff --git a/tests/api3.renderer.test.js b/tests/api3.renderer.test.js index 70401897025c..f92af5b7d72d 100644 --- a/tests/api3.renderer.test.js +++ b/tests/api3.renderer.test.js @@ -41,12 +41,14 @@ describe('API3 output renderers', function() { self.app = self.instance.app; self.env = self.instance.env; - self.url = '/api/v3/entries'; + self.col = 'entries'; + self.url = `/api/v3/${self.col}`; let authResult = await authSubject(self.instance.ctx.authorization.storage); self.subject = authResult.subject; self.token = authResult.token; + self.cache = self.instance.cacheMonitor; }); @@ -55,6 +57,16 @@ describe('API3 output renderers', function() { }); + beforeEach(() => { + self.cache.clear(); + }); + + + afterEach(() => { + self.cache.shouldBeEmpty(); + }); + + /** * Checks if all properties from obj1 are string identical in obj2 * (comparison of properties is made using toString()) @@ -139,7 +151,10 @@ describe('API3 output renderers', function() { } self.doc1json = await createDoc(self.doc1); + self.cache.nextShouldEql(self.col, self.doc1) + self.doc2json = await createDoc(self.doc2); + self.cache.nextShouldEql(self.col, self.doc2) }); @@ -262,7 +277,10 @@ describe('API3 output renderers', function() { } await deleteDoc(self.doc1.identifier); + self.cache.nextShouldDeleteLast(self.col) + await deleteDoc(self.doc2.identifier); + self.cache.nextShouldDeleteLast(self.col) }); }); diff --git a/tests/api3.update.test.js b/tests/api3.update.test.js index 481827b05d62..71f1c021192c 100644 --- a/tests/api3.update.test.js +++ b/tests/api3.update.test.js @@ -21,7 +21,7 @@ describe('API3 UPDATE', function() { eventType: 'Correction Bolus', insulin: 0.3 }; - + self.timeout(15000); @@ -41,13 +41,15 @@ describe('API3 UPDATE', function() { self.app = self.instance.app; self.env = self.instance.env; - self.url = '/api/v3/treatments'; + self.col = 'treatments' + self.url = `/api/v3/${self.col}`; let authResult = await authSubject(self.instance.ctx.authorization.storage); self.subject = authResult.subject; self.token = authResult.token; self.urlToken = `${self.url}/${self.validDoc.identifier}?token=${self.token.update}` + self.cache = self.instance.cacheMonitor; }); @@ -56,6 +58,16 @@ describe('API3 UPDATE', function() { }); + beforeEach(() => { + self.cache.clear(); + }); + + + afterEach(() => { + self.cache.shouldBeEmpty(); + }); + + it('should require authentication', async () => { let res = await self.instance.put(`${self.url}/FAKE_IDENTIFIER`) .expect(401); @@ -90,6 +102,7 @@ describe('API3 UPDATE', function() { .expect(201); res.body.should.be.empty(); + self.cache.nextShouldEql(self.col, self.validDoc) const lastModified = new Date(res.headers['last-modified']).getTime(); // Last-Modified has trimmed milliseconds @@ -113,6 +126,7 @@ describe('API3 UPDATE', function() { .expect(204); res.body.should.be.empty(); + self.cache.nextShouldEql(self.col, self.validDoc) const lastModified = new Date(res.headers['last-modified']).getTime(); // Last-Modified has trimmed milliseconds @@ -137,6 +151,7 @@ describe('API3 UPDATE', function() { .expect(204); res.body.should.be.empty(); + self.cache.nextShouldEql(self.col, doc) let body = await self.get(self.validDoc.identifier); body.should.containEql(doc); @@ -270,6 +285,8 @@ describe('API3 UPDATE', function() { .expect(204); res.body.should.be.empty(); + delete self.validDoc.srvModified; + self.cache.nextShouldEql(self.col, self.validDoc) }); @@ -278,6 +295,7 @@ describe('API3 UPDATE', function() { .expect(204); res.body.should.be.empty(); + self.cache.nextShouldDeleteLast(self.col) res = await self.instance.put(self.urlToken) .send(self.validDoc) diff --git a/tests/fixtures/api3/cacheMonitor.js b/tests/fixtures/api3/cacheMonitor.js new file mode 100644 index 000000000000..2c4180b01097 --- /dev/null +++ b/tests/fixtures/api3/cacheMonitor.js @@ -0,0 +1,79 @@ +'use strict'; + +/** + * Cache monitoring mechanism for tracking and checking cache updates. + * @param instance + * @constructor + */ +function CacheMonitor (instance) { + + const self = this; + + let operations = [] + , listening = false; + + instance.ctx.bus.on('data-update', operation => { + if (listening) { + operations.push(operation); + } + }); + + self.listen = () => { + listening = true; + return self; + } + + self.stop = () => { + listening = false; + return self; + } + + self.clear = () => { + operations = []; + return self; + } + + self.shouldBeEmpty = () => { + operations.length.should.equal(0) + } + + self.nextShouldEql = (col, doc) => { + operations.length.should.not.equal(0) + + const operation = operations.shift(); + operation.type.should.equal(col); + operation.op.should.equal('update'); + + if (doc) { + operation.changes.should.be.Array(); + operation.changes.length.should.be.eql(1); + const change = operation.changes[0]; + change.should.containEql(doc); + } + } + + self.nextShouldEqlLast = (col, doc) => { + self.nextShouldEql(col, doc); + self.shouldBeEmpty(); + } + + self.nextShouldDelete = (col, _id) => { + operations.length.should.not.equal(0) + + const operation = operations.shift(); + operation.type.should.equal(col); + operation.op.should.equal('remove'); + + if (_id) { + operation.changes.should.equal(_id); + } + } + + self.nextShouldDeleteLast = (col, _id) => { + self.nextShouldDelete(col, _id); + self.shouldBeEmpty(); + } + +} + +module.exports = CacheMonitor; diff --git a/tests/fixtures/api3/instance.js b/tests/fixtures/api3/instance.js index a7693ab3c404..beaa03da18d6 100644 --- a/tests/fixtures/api3/instance.js +++ b/tests/fixtures/api3/instance.js @@ -8,6 +8,7 @@ var fs = require('fs') , request = require('supertest') , websocket = require('../../../lib/server/websocket') , io = require('socket.io-client') + , CacheMonitor = require('./cacheMonitor') ; function configure () { @@ -54,6 +55,7 @@ function configure () { }; + self.bindSocket = function bindSocket (storageSocket, instance) { return new Promise(function (resolve, reject) { @@ -88,9 +90,9 @@ function configure () { /* * Create new web server instance for testing purposes */ - self.create = function createHttpServer ({ - apiSecret = 'this is my long pass phrase', - disableSecurity = false, + self.create = function createHttpServer ({ + apiSecret = 'this is my long pass phrase', + disableSecurity = false, useHttps = true, authDefaultRoles = '', enable = ['careportal', 'api'], @@ -129,6 +131,7 @@ function configure () { instance.baseUrl = `${useHttps ? 'https' : 'http'}://${instance.env.HOSTNAME}:${instance.env.PORT}`; self.addSecuredOperations(instance); + instance.cacheMonitor = new CacheMonitor(instance).listen(); websocket(instance.env, instance.ctx, instance.server); @@ -160,4 +163,4 @@ function configure () { return self; } -module.exports = configure(); \ No newline at end of file +module.exports = configure();