APIv3: Cache invalidation + refactoring

lib/api3/generic/collection.js

@@ -5,7 +5,8 @@ const apiConst = require('../const.json')
   , dateTools = require('../shared/dateTools')
   , opTools = require('../shared/operationTools')
   , stringTools = require('../shared/stringTools')
-  , CollectionStorage = require('../storage/mongoCollection')
+  , MongoCollectionStorage = require('../storage/mongoCollection')
+  , CachedCollectionStorage = require('../storage/mongoCachedCollection')
   , searchOperation = require('./search/operation')
   , createOperation = require('./create/operation')
   , readOperation = require('./read/operation')
@@ -26,13 +27,16 @@ function Collection ({ ctx, env, app, colName, storageColName, fallbackGetDate,
     fallbackDateField }) {
 
   const self = this;
-    
+
   self.colName = colName;
   self.fallbackGetDate = fallbackGetDate;
   self.dedupFallbackFields = app.get('API3_DEDUP_FALLBACK_ENABLED') ? dedupFallbackFields : [];
   self.autoPruneDays = app.setENVTruthy('API3_AUTOPRUNE_' + colName.toUpperCase());
   self.nextAutoPrune = new Date();
-  self.storage = new CollectionStorage(ctx, env, storageColName);
+
+  const baseStorage = new MongoCollectionStorage(ctx, env, storageColName);
+  self.storage = new CachedCollectionStorage(ctx, env, colName, baseStorage);
+
   self.fallbackDateField = fallbackDateField;
 
   self.mapRoutes = function mapRoutes () {
@@ -89,9 +93,9 @@ function Collection ({ ctx, env, app, colName, storageColName, fallbackGetDate,
   };
 
 
-  
+
   /**
-    * Fetch modified date from document (with possible fallback and back-fill to srvModified/srvCreated) 
+    * Fetch modified date from document (with possible fallback and back-fill to srvModified/srvCreated)
     * @param {Object} doc - document loaded from database
     */
   self.resolveDates = function resolveDates (doc) {
@@ -125,12 +129,12 @@ function Collection ({ ctx, env, app, colName, storageColName, fallbackGetDate,
    * in the background (asynchronously)
    * */
   self.autoPrune = function autoPrune () {
-    
+
     if (!stringTools.isNumberInString(self.autoPruneDays))
       return;
-    
+
     const autoPruneDays = parseFloat(self.autoPruneDays);
-    if (autoPruneDays <= 0) 
+    if (autoPruneDays <= 0)
       return;
 
     if (new Date() > self.nextAutoPrune) {
@@ -190,4 +194,4 @@ function Collection ({ ctx, env, app, colName, storageColName, fallbackGetDate,
   }
 }
 
-module.exports = Collection;
+module.exports = Collection;

lib/api3/generic/history/operation.js

@@ -27,13 +27,13 @@ async function history (opCtx, fieldsProjector) {
 
   if (filter !== null && limit !== null && projection !== null) {
 
-    const result = await col.storage.findMany(filter
+    const result = await col.storage.findMany({ filter
       , sort
       , limit
       , skip
       , projection
       , onlyValid
-      , logicalOperator);
+      , logicalOperator });
 
     if (!result)
       throw new Error('empty result');
@@ -150,4 +150,4 @@ function historyOperation (ctx, env, app, col) {
   };
 }
 
-module.exports = historyOperation;
+module.exports = historyOperation;

lib/api3/generic/search/operation.js

@@ -35,13 +35,12 @@ async function search (opCtx) {
 
 
   if (filter !== null && sort !== null && limit !== null && skip !== null && projection !== null) {
-
-    const result = await col.storage.findMany(filter
+    const result = await col.storage.findMany({ filter
       , sort
       , limit
       , skip
       , projection
-      , onlyValid);
+      , onlyValid });
 
     if (!result)
       throw new Error('empty result');
@@ -76,4 +75,4 @@ function searchOperation (ctx, env, app, col) {
   };
 }
 
-module.exports = searchOperation;
+module.exports = searchOperation;

lib/api3/storage/mongoCachedCollection/index.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const _ = require('lodash')
+
+/**
+ * Storage implementation which wraps mongo baseStorage with caching
+ * @param {Object} ctx
+ * @param {Object} env
+ * @param {string} colName - name of the collection in mongo database
+ * @param {Object} baseStorage - wrapped mongo storage implementation
+ */
+function MongoCachedCollection (ctx, env, colName, baseStorage) {
+
+  const self = this;
+
+  self.colName = colName;
+
+  self.identifyingFilter = baseStorage.identifyingFilter;
+
+  self.findOne = (...args) => baseStorage.findOne(...args);
+
+  self.findOneFilter = (...args) => baseStorage.findOneFilter(...args);
+
+  self.findMany = (...args) => baseStorage.findMany(...args);
+
+
+  self.insertOne = async (doc) => {
+    const result = await baseStorage.insertOne(doc, { normalize: false });
+
+    if (cacheSupported()) {
+      updateInCache([doc]);
+    }
+
+    if (doc._id) {
+      delete doc._id;
+    }
+    return result;
+  }
+
+
+  self.replaceOne = async (identifier, doc) => {
+    const result = await baseStorage.replaceOne(identifier, doc);
+
+    if (cacheSupported()) {
+      const rawDocs = await baseStorage.findOne(identifier, null, { normalize: false })
+      updateInCache([rawDocs[0]])
+    }
+
+    return result;
+  }
+
+
+  self.updateOne = async (identifier, setFields) => {
+    const result = await baseStorage.updateOne(identifier, setFields);
+
+    if (cacheSupported()) {
+      const rawDocs = await baseStorage.findOne(identifier, null, { normalize: false })
+
+      if (rawDocs[0].isValid === false) {
+        deleteInCache(rawDocs)
+      }
+      else {
+        updateInCache([rawDocs[0]])
+      }
+    }
+
+    return result;
+  }
+
+  self.deleteOne = async (identifier) => {
+    let invalidateDocs
+    if (cacheSupported()) {
+      invalidateDocs = await baseStorage.findOne(identifier, { _id: 1 }, { normalize: false })
+    }
+
+    const result = await baseStorage.deleteOne(identifier);
+
+    if (cacheSupported()) {
+      deleteInCache(invalidateDocs)
+    }
+
+    return result;
+  }
+
+  self.deleteManyOr = async (filter) => {
+    let invalidateDocs
+    if (cacheSupported()) {
+      invalidateDocs = await baseStorage.findMany({ filter,
+        limit: 1000,
+        skip: 0,
+        projection: { _id: 1 },
+        options: { normalize: false } });
+    }
+
+    const result = await baseStorage.deleteManyOr(filter);
+
+    if (cacheSupported()) {
+      deleteInCache(invalidateDocs)
+    }
+
+    return result;
+  }
+
+  self.version = (...args) => baseStorage.version(...args);
+
+  self.getLastModified = (...args) => baseStorage.getLastModified(...args);
+
+  function cacheSupported () {
+    return ctx.cache
+      && ctx.cache[colName]
+      && _.isArray(ctx.cache[colName]);
+  }
+
+  function updateInCache (doc) {
+    if (doc && doc.isValid === false) {
+      deleteInCache([doc._id])
+    }
+    else {
+      ctx.bus.emit('data-update', {
+        type: colName
+        , op: 'update'
+        , changes: doc
+      });
+    }
+  }
+
+  function deleteInCache (docs) {
+    let changes
+    if (_.isArray(docs)) {
+      if (docs.length === 0) {
+        return
+      }
+      else if (docs.length === 1 && docs[0]._id) {
+        const _id = docs[0]._id.toString()
+        changes = [ _id ]
+      }
+    }
+
+    ctx.bus.emit('data-update', {
+      type: colName
+      , op: 'remove'
+      , changes
+    });
+  }
+}
+
+module.exports = MongoCachedCollection;

lib/api3/storage/mongoCollection/find.js

@@ -10,8 +10,9 @@ const utils = require('./utils')
  * @param {Object} col
  * @param {string} identifier
  * @param {Object} projection
+ * @param {Object} options
  */
-function findOne (col, identifier, projection) {
+function findOne (col, identifier, projection, options) {
 
   return new Promise(function (resolve, reject) {
 
@@ -25,7 +26,9 @@ function findOne (col, identifier, projection) {
         if (err) {
           reject(err);
         } else {
-          _.each(result, utils.normalizeDoc);
+          if (!options || options.normalize !== false) {
+            _.each(result, utils.normalizeDoc);
+          }
           resolve(result);
         }
       });
@@ -38,8 +41,9 @@ function findOne (col, identifier, projection) {
  * @param {Object} col
  * @param {Object} filter specific filter
  * @param {Object} projection
+ * @param {Object} options
  */
-function findOneFilter (col, filter, projection) {
+function findOneFilter (col, filter, projection, options) {
 
   return new Promise(function (resolve, reject) {
 
@@ -51,7 +55,9 @@ function findOneFilter (col, filter, projection) {
         if (err) {
           reject(err);
         } else {
-          _.each(result, utils.normalizeDoc);
+          if (!options || options.normalize !== false) {
+            _.each(result, utils.normalizeDoc);
+          }
           resolve(result);
         }
       });
@@ -62,23 +68,25 @@ function findOneFilter (col, filter, projection) {
 /**
  * Find many documents matching the filtering criteria
  */
-function findMany (col, filterDef, sort, limit, skip, projection, onlyValid, logicalOperator = 'and') {
-
+function findMany (col, args) {
+  const logicalOperator = args.logicalOperator || 'and';
   return new Promise(function (resolve, reject) {
 
-    const filter = utils.parseFilter(filterDef, logicalOperator, onlyValid);
+    const filter = utils.parseFilter(args.filter, logicalOperator, args.onlyValid);
 
     col.find(filter)
-      .sort(sort)
-      .limit(limit)
-      .skip(skip)
-      .project(projection)
+      .sort(args.sort)
+      .limit(args.limit)
+      .skip(args.skip)
+      .project(args.projection)
       .toArray(function mongoDone (err, result) {
 
         if (err) {
           reject(err);
         } else {
-          _.each(result, utils.normalizeDoc);
+          if (!args.options || args.options.normalize !== false) {
+            _.each(result, utils.normalizeDoc);
+          }
           resolve(result);
         }
       });
@@ -90,4 +98,4 @@ module.exports = {
   findOne,
   findOneFilter,
   findMany
-};
+};

lib/api3/storage/mongoCollection/index.js

@@ -18,7 +18,7 @@ function MongoCollection (ctx, env, colName) {
 
   self.col = ctx.store.collection(colName);
 
-  ctx.store.ensureIndexes(self.col, [ 'identifier', 
+  ctx.store.ensureIndexes(self.col, [ 'identifier',
     'srvModified',
     'isValid'
   ]);
@@ -64,10 +64,10 @@ function MongoCollection (ctx, env, colName) {
 
 
   /**
-   * Get timestamp (e.g. srvModified) of the last modified document 
+   * Get timestamp (e.g. srvModified) of the last modified document
    */
   self.getLastModified = function getLastModified (fieldName) {
-    
+
     return new Promise(function (resolve, reject) {
 
       self.col.find()
@@ -87,4 +87,4 @@ function MongoCollection (ctx, env, colName) {
   }
 }
 
-module.exports = MongoCollection;
+module.exports = MongoCollection;