diff --git a/.eslintrc.js b/.eslintrc.js index 5a21701..8afa3f5 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -32,6 +32,62 @@ module.exports = { "semi": [ "warn", "always" + ], + "key-spacing": [ + "error" + ], + "func-call-spacing": [ + "error", + "never" + ], + "block-spacing": [ + "error" + ], + "array-bracket-spacing": [ + "error" + ], + "keyword-spacing": [ + "error" + ], + "semi-spacing": [ + "error" + ], + "arrow-spacing": [ + "error" + ], + "no-tabs": [ + "error" + ], + "space-before-function-paren": [ + "error", + "never" + ], + "no-multiple-empty-lines": [ + "error", + { "max": 1 } + ], + "padded-blocks": [ + "error", + "never" + ], + "padding-line-between-statements": [ + "error", + { + "blankLine": "always", + "prev": "*", + "next": "if" + } + ], + "sort-imports": [ + "error" + ], + "comma-dangle": [ + "error", + "never" + ], + "object-curly-spacing": [ + "error", + "always" ] } }; \ No newline at end of file diff --git a/.gitignore b/.gitignore index 88d5b21..9bbece1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ .cert* .env* data* -node_modules \ No newline at end of file +node_modules diff --git a/Gruntfile.js b/Gruntfile.js index 819b07f..12cf24e 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -32,7 +32,7 @@ module.exports = function(grunt) { jsdoc: { build: { options: { - configure : '.jsdoc.json' + configure: '.jsdoc.json' } } }, diff --git a/app/.DS_Store b/app/.DS_Store new file mode 100644 index 0000000..9ff5c33 Binary files /dev/null and b/app/.DS_Store differ diff --git a/app/controllers/item.js b/app/controllers/item.js index c4c703f..435d937 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -25,12 +25,13 @@ var kue = require('kue'); var logger = require('app/lib/logger'); var mime = require('app/lib/mime'); var request = require('app/lib/request'); +var SourceContentType = require('app/models/sourceContentType'); +var templateCompiler = require('es6-template-strings'); var Url = require('url'); var urlRegex = require('app/lib/urlRegex'); var UserSourceAuth = require('app/models/userSourceAuth'); var UserStorageAuth = require('app/models/userStorageAuth'); var validateParams = require('app/lib/validateParams'); - var queue = kue.createQueue(); queue.process('storeItemData', function(queueJob, done) { @@ -82,6 +83,7 @@ queue.on('error', (error) => { /** * Callback resource found at URL. + *That is: get resource from passed URL and pass to callback (done) * @param {string} url - URL of resource with extension that corresponds to a supported media type. * @param {module:controllers/item~resourceCallback} done */ @@ -128,7 +130,7 @@ module.exports.getResource = function(url, done) { var resource; - switch(mediaType) { + switch (mediaType) { case 'image/jpeg': resource = new Buffer(body); break; @@ -149,12 +151,12 @@ module.exports.getResource = function(url, done) { }; async.waterfall([ - validate, + validate, setupLog, getResource ], function(error, resource) { if (error) { - log('error', 'Item controller failed to get resource', { error: error }); + log('error', 'Item controller failed to get resource', { error: error }); } done(error, resource); @@ -211,12 +213,12 @@ module.exports.itemDataObjectsFromPage = function(page, source, contentType) { /** * Return URL for making a GET request for items from source. * @param {Object} source - Source from which to retrieve items. - * @param {Object} contentType - ContentType of items. + * @param {Object} contentType - contentType of items. * @param {Object} userSourceAuth - UserSourceAuth used to make request. * @param {Object} pagination - Pagination used to make request. * @returns {string} URL for making a GET request */ -module.exports.itemsGetUrl = function(source, contentType, userSourceAuth, pagination) { +module.exports.itemsGetUrl = function(source, contentType, userSourceAuth, pagination,done) { validateParams([{ name: 'source', variable: source, required: true, requiredProperties: ['host'] }, { @@ -227,18 +229,37 @@ module.exports.itemsGetUrl = function(source, contentType, userSourceAuth, pagin name: 'pagination', variable: pagination }]); - return source.itemsGetUrl({ - accessToken: userSourceAuth.sourceToken, - apiVersion: source.apiVersion, - contentTypePluralCamelName: contentType.pluralCamelName(), - contentTypePluralLowercaseName: contentType.pluralLowercaseName(), - host: source.host, - limit: source.itemsLimit, - maxId: (typeof pagination !== 'undefined' && pagination.maxId) ? pagination.maxId : undefined, - offset: (typeof pagination !== 'undefined' && pagination.offset) ? pagination.offset : 0, - next: (typeof pagination !== 'undefined' && pagination.next) ? pagination.next : undefined, - sourceName: source.name - }); + debug('getItemURL, source.authScope = %s', source.authScope); + + var property_next = (typeof pagination !== 'undefined' && pagination.next) ? pagination.next : undefined; + + if (property_next) { + return done(undefined, property_next); + } else { + SourceContentType.findOne({ + contentType: contentType.id, + source: source.id + }, function(error, sourceContentType) { + if (error) { + return done(error); + } else { + var urlToReturn = templateCompiler(sourceContentType.itemsGetUrlTemplate, + { + sourceToken: userSourceAuth.sourceToken, + apiVersion: source.apiVersion, + contentTypePluralCamelName: sourceContentType.contentType.pluralCamelName(), + contentTypePluralLowercaseName: sourceContentType.contentType.pluralLowercaseName(), + sourceHost: source.host, + sourceItemsLimit: source.itemsLimit, + maxId: (typeof pagination !== 'undefined' && pagination.maxId) ? pagination.maxId : undefined, + offset: (typeof pagination !== 'undefined' && pagination.offset) ? pagination.offset : 0, + next: property_next, + sourceName: source.name + }); + return done(undefined, urlToReturn); + } + }); + } }; /** @@ -267,8 +288,6 @@ module.exports.totalItemsAvailableFromPage = function(page, source, contentType) return total; }; - - /** * Returns error from items page if error exists within. * @param {Object} page - Items page. @@ -294,7 +313,6 @@ module.exports.itemsPageError = function(page) { } }; - /** * Returns pagination for next items page after current items page. * @param {Object} page - Current items page. @@ -310,9 +328,9 @@ module.exports.itemsPageNextPagination = function(page, pagination, contentType) }]); var nextPagination; - + debug.start('itemsPageNextPagination (pagination: %o)', pagination); - + if (page.response && page.response[contentType.pluralLowercaseName()] && page.response[contentType.pluralLowercaseName()].items && page.response[contentType.pluralLowercaseName()].items.length) { if (pagination && pagination.offset) { nextPagination = { offset: pagination.offset + page.response[contentType.pluralLowercaseName()].items.length }; @@ -395,16 +413,20 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo done(); }; - var storeAllForUserStorageSourceContentType = function(contentType, done) { - module.exports.storeAllForUserStorageSourceContentType(user, source, storage, contentType, job, done); + var storeAllForUserStorageContentType = function(contentType, done) { + module.exports.storeAllForUserStorageContentType(user, source, storage, contentType, job, done); }; - var storeAllItems = function(done) { - debug.start('storeAllItems (contentTypes: %s)', source.contentTypes.length); - async.eachSeries(source.contentTypes, storeAllForUserStorageSourceContentType, done); + var getContentTypes = function(done) { + source.getContentTypes(done); }; - async.waterfall([validate, setupLog, storeAllItems], function(error) { + var storeAllItems = function(contentTypes, done) { + debug.start('storeAllItems (contentTypes: %s)', contentTypes.length); + async.eachSeries(contentTypes, storeAllForUserStorageContentType, done); + }; + + async.waterfall([validate, setupLog, getContentTypes, storeAllItems], function(error) { if (error) { log('error', 'Item controller failed to store all items', { error: error.message }); } else { @@ -422,11 +444,12 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo * @param {User} user - User for which to retrieve items from source and store them in storage. * @param {Source} source - Source from which to retrieve items. * @param {Storage} storage - Storage within which to store items. - * @param {ContentType} contentType - ContentType of which to retrieve items. + * @param {ContentType} contentType - contentType of which to retrieve items. * @param {Job} [job] - Job for which to store items. * @param {callback} done */ -module.exports.storeAllForUserStorageSourceContentType = function(user, source, storage, contentType, job, done) { +module.exports.storeAllForUserStorageContentType = function(user, source, storage, contentType, job, done) { + debug('storeAllForUserStorageContentType, source = %s, storage = %s, contentType = %s', source, storage, contentType); var log = logger.scopedLog(); var validate = function(done) { @@ -442,7 +465,7 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, }; var setupLog = function(done) { - debug.start('storeAllForUserStorageSourceContentType'); + debug.start('storeAllForUserStorageContentType'); log = logger.scopedLog({ user: user.id, @@ -474,10 +497,10 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, async.series([validate, setupLog, storeAllItems], function(error) { if (error) { - debug.error('storeAllForUserStorageSourceContentType (message: %s)', error.message); + debug.error('storeAllForUserStorageContentType (message: %s)', error.message); log('error', 'Item controller failed to store all items', { error: error }); } else { - debug.success('storeAllForUserStorageSourceContentType'); + debug.success('storeAllForUserStorageContentType'); log('milestone', 'Item controller stored all items', { error: error }); } @@ -492,7 +515,7 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, * @param {User} user - User for which to retrieve items from source and store them in storage. * @param {Source} source - Source from which to retrieve items. * @param {Storage} storage - Storage within which to store items. - * @param {ContentType} contentType - ContentType of which to retrieve items. + * @param {ContentType} contentType - contentType of which to retrieve items. * @param {Object} pagination – Object containing pagination information. * @param {Job} [job] - Job for which to store items. * @param {callback} done @@ -543,8 +566,15 @@ module.exports.storeItemsPage = function(user, source, storage, contentType, pag }); }; - var getItemsPageResource = function(done) { - module.exports.getResource(module.exports.itemsGetUrl(source, contentType, userSourceAuth, pagination), done); + // get the page of date from URL + // whcih will be converted INTO items + + var getItemsUrlFromTemplate = function(done) { + module.exports.itemsGetUrl(source, contentType, userSourceAuth, pagination,done); + }; + + var getItemsPageResource = function(url, done) { + module.exports.getResource(url, done); }; var getItemDataObjects = function(resource, done) { @@ -574,7 +604,8 @@ module.exports.storeItemsPage = function(user, source, storage, contentType, pag async.mapSeries(itemDataObjects, function(itemDataObject, done) { count++; debug('persistItemDataObject #%s', count); - + + // this creates the Item mongo data document module.exports.persistItemDataObject(itemDataObject, { user: user, storage: storage, @@ -599,7 +630,8 @@ module.exports.storeItemsPage = function(user, source, storage, contentType, pag if (job) { jobAttributes.jobId = job.id; } - + // this is where the actual jobs queue "items" are created + /// where the magic happens… var queueJob = queue.create('storeItemData', jobAttributes).save((error) => { if (error) { debug.error('queueJob %s failed to queue: %s', queueJob.id, error.message); @@ -620,6 +652,7 @@ module.exports.storeItemsPage = function(user, source, storage, contentType, pag validate, setupLog, findUserSourceAuth, + getItemsUrlFromTemplate, getItemsPageResource, getItemDataObjects, persistItemDataObjects, @@ -644,6 +677,8 @@ module.exports.storeItemsPage = function(user, source, storage, contentType, pag * @param {Object} relationships - Relationships to use for persistence of item with itemDataObject. * @param {function} done - Error-first callback function expecting Item as second parameter. */ + +// this creates the Item _about_ the data we're about to store, and saves this MongoDBx module.exports.persistItemDataObject = function(itemDataObject, relationships, done) { var conditions; var log = logger.scopedLog(); @@ -652,7 +687,10 @@ module.exports.persistItemDataObject = function(itemDataObject, relationships, d validateParams([{ name: 'itemDataObject', variable: itemDataObject, required: true, requiredProperties: ['id'] }, { - name: 'relationships', variable: relationships, required: true, requiredProperties: ['user', 'storage', 'source', 'contentType'] + name: 'relationships', + variable: relationships, + required: true, + requiredProperties: ['user', 'storage', 'source', 'contentType'] }], done); }; @@ -664,7 +702,7 @@ module.exports.persistItemDataObject = function(itemDataObject, relationships, d storage: relationships.storage.id, source: relationships.source.id, contentType: relationships.contentType.id, - sourceItem: itemDataObject.id, + sourceItem: itemDataObject.id }; done(); }; @@ -753,7 +791,7 @@ module.exports.persistItemDataObject = function(itemDataObject, relationships, d if (error) { log('error', 'Item controller failed to persist item data object', { error: error }); } else { - debug.success('persistItemDataObject'); + debug.success('persistItemDataObject', item); } done(error, item); @@ -770,6 +808,7 @@ module.exports.persistItemDataObject = function(itemDataObject, relationships, d * @param {Job} [job] - Job for which to store items. * @param {callback} done */ +// item is newly created item, data is the data for that item (from source) module.exports.storeItemData = function(item, data, job, done) { var log = logger.scopedLog(); @@ -793,8 +832,9 @@ module.exports.storeItemData = function(item, data, job, done) { done(error); }); }; - + // ?? what is going on here?? var storeFile = function(done) { + debug('storeFile : item.user = ', item.user); module.exports.storeFile(item.user, item.storage, item.storagePath, data, (error, storeFileResult) => { if (error) { debug.error('storeFile item %s, error %o, storeFileResult %o', item.id, error, storeFileResult); @@ -914,6 +954,7 @@ module.exports.storeFile = function(user, storage, path, data, done) { var prepareData = function(done) { debug.start('storeFile (path: %s)', path); + if (!(data instanceof Buffer)) { data = JSON.stringify(data); } @@ -936,7 +977,7 @@ module.exports.storeFile = function(user, storage, path, data, done) { storage: storage.id, user: user.id }, function(error, userStorageAuth) { - if(!error && !userStorageAuth) { + if (!error && !userStorageAuth) { error = new Error('Failed to retrieve userStorageAuth'); } @@ -953,6 +994,7 @@ module.exports.storeFile = function(user, storage, path, data, done) { debug('storeFile:options %o', options); + // what is going on t is ????? request.post(options, function(error, res, body) { if (!error) { error = request.statusCodeError(res.statusCode); @@ -961,7 +1003,7 @@ module.exports.storeFile = function(user, storage, path, data, done) { if (!error) { body = JSON.parse(body); } - + debug('storeFile body %o, error %o', body, error); done(error, body); @@ -976,7 +1018,7 @@ module.exports.storeFile = function(user, storage, path, data, done) { storeFile ], function(error, responseBody) { if (error) { - log('error', 'Item controller failed to store file', { error: error.message, responseBody: responseBody }); + log('error', 'Item controller failed to store file', { error: error.message, responseBody: responseBody }); } done(error, responseBody); diff --git a/app/lib/.DS_Store b/app/lib/.DS_Store new file mode 100644 index 0000000..1cfcb50 Binary files /dev/null and b/app/lib/.DS_Store differ diff --git a/app/lib/assertions/it.js b/app/lib/assertions/it.js index e8cbb28..2271f0c 100644 --- a/app/lib/assertions/it.js +++ b/app/lib/assertions/it.js @@ -64,7 +64,6 @@ module.exports = function(description, assertion) { return description; }; - return function(subjectName, subject, tests) { if (!tests) { tests = subject; diff --git a/app/lib/jsonapi.js b/app/lib/jsonapi.js index 950b66c..5e9117e 100644 --- a/app/lib/jsonapi.js +++ b/app/lib/jsonapi.js @@ -12,7 +12,6 @@ var logger = require('app/lib/logger'); var models = require('app/models'); var ObjectId = require('mongoose').Types.ObjectId; var validateParams = require('./validateParams'); - var jsonapi = {}; /** @@ -29,6 +28,7 @@ jsonapi.resourceObjectFromDocument = function(document) { var Model = models[document.modelId()]; var attributes = document.toObject(); + debug('attributes %O ', attributes); delete attributes.id; var relationships = {}; @@ -140,6 +140,7 @@ jsonapi.allowed = function(model, method) { */ jsonapi.compiledQueryConditions = function myself(req, conditions, model, method, done) { this.modelQueryConditions(req, model, method, (error, modelConditions) => { + debug('jsonapi.compiledQueryConditions : ', modelConditions, conditions); done(error, Object.assign({}, modelConditions, conditions)); }); }; @@ -257,7 +258,10 @@ jsonapi.routeModelGetObjectResource = function(app, Model) { async.waterfall([getConditions, findOne], (error, document) => { if (error) { - logger.error('Resource router failed to query for object', { model: Model.modelName, error: error.message }); + logger.error('Resource router failed to query for object', { + model: Model.modelName, + error: error.message + }); this.sendError(res); } else if (!document) { this.sendNotFound(res); @@ -278,7 +282,7 @@ jsonapi.routeModelGetObjectsResource = function(app, Model) { var compileConditions = (done) => { var conditions = {}; var filter = req.query.filter ? req.query.filter : {}; - + try { if (filter.relationships) { Object.keys(filter.relationships).forEach((modelName) => { @@ -323,7 +327,10 @@ jsonapi.routeModelGetObjectsResource = function(app, Model) { async.waterfall([compileConditions, executeQuery], function(error, documents) { if (error) { - logger.error('Resource router failed to query for objects', { model: Model.modelName, error: error.message }); + logger.error('Resource router failed to query for objects', { + model: Model.modelName, + error: error.message + }); jsonapi.sendError(res, error, 400); } else { jsonapi.sendDocuments(res, documents); @@ -411,8 +418,11 @@ jsonapi.routeModelPatchObjectResource = function(app, Model) { * @param {Object} app - Express app * @param {Object} Model - Mongoose model */ + jsonapi.routeModelPostObjectResource = function(app, Model) { - this.routeModelResource(app, Model, 'post', '/'+ _.kebabCase(Model.modelType()), (req, res) => { + debug('routeModelPostObjectResource -- Model = ', Model.collection.collectionName); + + this.routeModelResource(app, Model, 'post', '/' + _.kebabCase(Model.modelType()), (req, res) => { /** * Validates all available attributes (TODO: and relationships) */ @@ -471,6 +481,8 @@ jsonapi.routeModelPostObjectResource = function(app, Model) { * Executes any available post-POST routine available for Model */ var executePostRoutine = (document, done) => { + debug('executePostRoutine... ', Model.collection.collectionName); + if (Model.jsonapi.post && Model.jsonapi.post.post) { Model.jsonapi.post.post(req, res, document, function(error) { done(error, document); @@ -510,7 +522,11 @@ jsonapi.routeModelPostObjectResource = function(app, Model) { * @param {function} done - Express route callback expecting req and res as parameters */ jsonapi.routeModelResource = function(app, model, method, path, done) { - if (!model.jsonapi || !model.jsonapi[method]) { return; } + debug('jsonapi.routeModelResource -- method = %s, path = %s', method, path); + + if (!model.jsonapi || !model.jsonapi[method]) { + return; + } var validateRequestBody = false; @@ -662,7 +678,7 @@ jsonapi.sendError = function(res, error, status) { } // Convert object of errors to array if needed - if(typeof errors === 'object' && !Array.isArray(errors)) { + if (typeof errors === 'object' && !Array.isArray(errors)) { errors = Object.keys(errors).map(function(key) { return errors[key]; }); @@ -735,14 +751,16 @@ jsonapi.routeModelResources = function() { }); }); + // default response app.get('/', (req, res) => { this.sendResponseDocument(res); }); // Route requests for each model with Mongoose compatability and jsonapi configuration + // that is: set up the routes for specific paths Object.keys(models).forEach((key) => { var model = models[key]; - + if (model.modelName && model.jsonapi) { this.routeModelGetObjectsResource(app, model); this.routeModelGetObjectResource(app, model); @@ -810,7 +828,7 @@ jsonapi.saveRelationshipsToDocument = function(document, relationships, done) { var saveRelationshipsToDocument = function(done) { var Model = models[document.modelId()]; - + async.forEachOf(relationships, function(relationship, relationshipName, done) { var validateRelationship = function(done) { var errors = []; @@ -926,7 +944,7 @@ jsonapi.validateQueryData = function(req, data, model, method, done) { Object.keys(conditions).forEach(function(key) { var isEqualObjectId; - + try { isEqualObjectId = ObjectId(_.get(data, `relationships.${key}.data.id`)).equals(conditions[key]); } catch (error) { diff --git a/app/lib/mailer.js b/app/lib/mailer.js index 4f02d36..db741cf 100644 --- a/app/lib/mailer.js +++ b/app/lib/mailer.js @@ -5,6 +5,7 @@ var sendGridTransport = require('nodemailer-sendgrid-transport'); var stubTransport = require('nodemailer-stub-transport'); var mailer = { + // eslint-disable-next-line no-useless-escape emailRegex: /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ }; diff --git a/app/lib/mongoose.js b/app/lib/mongoose.js index 8d796fa..822a404 100644 --- a/app/lib/mongoose.js +++ b/app/lib/mongoose.js @@ -33,7 +33,7 @@ mongoose.transform = function(doc, ret) { } // Convert under_scores to camelCase - newKey = key.replace(/_([a-z])/g, function (g) { return g[1].toUpperCase(); }); + newKey = key.replace(/_([a-z])/g, function(g) { return g[1].toUpperCase(); }); if (newKey != key) { ret[newKey] = ret[key]; diff --git a/app/lib/tasks/grunt-repopulate-collections.js b/app/lib/tasks/grunt-repopulate-collections.js index de22ee1..b80b973 100644 --- a/app/lib/tasks/grunt-repopulate-collections.js +++ b/app/lib/tasks/grunt-repopulate-collections.js @@ -38,7 +38,6 @@ module.exports = function(grunt) { if (!relatedDocument) { done(new Error('Unable to find related document referenced by resourceObject')); } else { - if (toMany) { if (!document[relationshipName]) { document[relationshipName] = []; diff --git a/app/lib/urlRegex.js b/app/lib/urlRegex.js index 7637e59..141cf7f 100644 --- a/app/lib/urlRegex.js +++ b/app/lib/urlRegex.js @@ -1,4 +1,5 @@ /** * Regex to validate URLs */ +// eslint-disable-next-line no-useless-escape module.exports = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%_\+.~#?&//=]{2,256}\.[a-z]{2,4}\b(\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?/; \ No newline at end of file diff --git a/app/lib/warehouse.js b/app/lib/warehouse.js index 886e9bf..c949b10 100644 --- a/app/lib/warehouse.js +++ b/app/lib/warehouse.js @@ -206,7 +206,7 @@ module.exports = { one: function(modelId, overwriteProperties) { if (ObjectId.isValid(overwriteProperties)) { overwriteProperties = { - _id : overwriteProperties + _id: overwriteProperties }; } diff --git a/app/models/.DS_Store b/app/models/.DS_Store new file mode 100644 index 0000000..3187162 Binary files /dev/null and b/app/models/.DS_Store differ diff --git a/app/models/contactVerificationRequest.js b/app/models/contactVerificationRequest.js index 53d819e..f72d2de 100644 --- a/app/models/contactVerificationRequest.js +++ b/app/models/contactVerificationRequest.js @@ -95,7 +95,6 @@ module.exports = ContactVerificationRequest = modelFactory.new('ContactVerificat _id: ObjectId(req.body.data.id), code: req.body.data.attributes.code }, (error, contactVerificationRequest) => { - if (!error && !contactVerificationRequest) { error = new Error('No contactVerificationRequest found with id and code'); } @@ -108,7 +107,7 @@ module.exports = ContactVerificationRequest = modelFactory.new('ContactVerificat }); } }); - }, + } }, post: { allowed: 'public', diff --git a/app/models/item.js b/app/models/item.js index f9f6ab7..2e9b941 100644 --- a/app/models/item.js +++ b/app/models/item.js @@ -10,7 +10,7 @@ var queryConditions = require('./queryConditions'); var sanitizeFilename = require('sanitize-filename'); var convertToFilename = function(content) { - return _.toLower(emojiStrip(sanitizeFilename(content).replace(/[^\x00-\x7F]/g, '').replace('.','').replace('-', ' ').replace(/ {2}/g, ' ').replace(/ +/g, '-').replace(/–|—+/g, '-'))); + return _.toLower(emojiStrip(sanitizeFilename(content).replace(/[^\x00-\x7F]/g, '').replace('.', '').replace('-', ' ').replace(/ {2}/g, ' ').replace(/ +/g, '-').replace(/–|—+/g, '-'))); }; /** diff --git a/app/models/notificationRequest.js b/app/models/notificationRequest.js index 0fd65b2..745d509 100644 --- a/app/models/notificationRequest.js +++ b/app/models/notificationRequest.js @@ -1,4 +1,4 @@ - /** +/** * NotificationRequest model * @module */ diff --git a/app/models/source.js b/app/models/source.js index 6578d43..fd4cf2c 100644 --- a/app/models/source.js +++ b/app/models/source.js @@ -3,9 +3,11 @@ * @module */ +var debug = require('app/lib/debug')('syncServer:models/source.js'); +var templateCompiler = require('es6-template-strings'); var modelFactory = require('app/factories/model'); var nameMethods = require('./methods/name'); -var templateCompiler = require('es6-template-strings'); +var SourceContentType = require('app/models/sourceContentType'); var methods = Object.assign({ itemDataObjectsFromPagePath: function(contentType) { @@ -22,13 +24,24 @@ var methods = Object.assign({ }); }, - itemsGetUrl: function(properties) { - if (properties.next) { - return properties.next; - } + /** + * returns the contentTypes associated with this source + * @param done {Function} A callback that will be passed a the error (Error) and optionally the contentTypes results + */ + getContentTypes: function(done) { + debug(this); + debug('this.id = %O',this.id); - return templateCompiler(this.itemsGetUrlTemplate, properties); + SourceContentType.find({ source: this._id }, function(err, sourceContentTypes) { + if (err) { + return done(err); + } else { + debug('sourceContentTypes = %O', sourceContentTypes); + done(err, sourceContentTypes.map((sourceContentType) => sourceContentType.contentType)); + } + }); } + }, nameMethods); /** @@ -37,14 +50,12 @@ var methods = Object.assign({ * @property {number} apiVersion - Version of API to use for pulling items from source * @property {string=} clientId - OAuth 2.0 client ID * @property {string=} clientSecret - OAuth 2.0 client secret - * @property {module:models/contentType~ContentType[]} contentTypes - ContentTypes supported by source * @property {boolean} [itemStorageEnabled=false] - Whether source is enabled for storing items in storage * @property {string} [host] - Host URL for source (e.g. "api.foursquare.com") * @property {number} [itemsLimit=25] - Maximum number of items to pull from source in a single page request * @property {string} [logoGlyphPath] - URL path to logo glyph image file on host (e.g. "/images/logos/foursquare-glyph.svg") * @property {string} name - Name of source (e.g. "foursquare") * @property {string} [passportStrategy] - Strategy for Passport module (e.g. "passport-foursquare") - * @property {string} [itemsGetUrlTemplate=https://${host}/${contentTypePluralCamelName}?access_token=${accessToken}&limit=${limit}&offset=${offset}] - String template used to generate URLs for GET requests for items on source * @property {string} [itemDataObjectsFromPagePathTemplate=data] - String template used to generate object paths to itemDataObjects found within pages returned from source * @property {string} [totalItemsAvailableFromPagePathTemplate=response.${contentTypePluralCamelName}.count] - String template used to generate object paths to value representing total items available for contentType within pages returned from source */ @@ -53,9 +64,7 @@ module.exports = modelFactory.new('Source', { authScope: Array, clientId: String, clientSecret: String, - contentTypes: [{ ref: 'ContentType' }], itemDataObjectsFromPagePathTemplate: { type: String, default: 'data' }, - itemsGetUrlTemplate: { type: String, default: 'https://${host}/${contentTypePluralCamelName}?access_token=${accessToken}&limit=${limit}&offset=${offset}' }, itemStorageEnabled: { type: Boolean, default: false }, host: String, itemsLimit: { type: Number, default: 25 }, diff --git a/app/models/sourceContentType.js b/app/models/sourceContentType.js new file mode 100644 index 0000000..8108572 --- /dev/null +++ b/app/models/sourceContentType.js @@ -0,0 +1,25 @@ +/** + * sourceContentType model + * @module + */ + +var modelFactory = require('app/factories/model'); +var nameMethods = require('./methods/name'); + +/** + * This model will related contentTypes to sources and when documents are created for it, + * they will imply that items for the given contentType + * can be imported from the source (e.g. "photos" can be imported from "Facebook"). + * @class sourceContentType + * @property {module:models/source~Source} source - source for this sourceContentType + * @property {module:models/contentType~ContentType} contentType for this sourceContentType + */ +module.exports = modelFactory.new('SourceContentType', { + contentType: { ref: 'ContentType',required: true }, + itemsGetUrlTemplate: { type: String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}' }, + source: { ref: 'Source',required: true } +}, { + jsonapi: { + get: 'public' + } +}, nameMethods); \ No newline at end of file diff --git a/app/models/storage.js b/app/models/storage.js index 2cc5309..d6eb6a7 100644 --- a/app/models/storage.js +++ b/app/models/storage.js @@ -69,7 +69,7 @@ module.exports = modelFactory.new('Storage', { */ itemPutUrl: function(path, userStorageAuth) { validateParams([{ - name: 'path', variable: path, required: true, requiredType: 'string', + name: 'path', variable: path, required: true, requiredType: 'string' }, { name: 'userStorageAuth', variable: userStorageAuth, required: true, requiredProperties: ['storageToken'] }]); diff --git a/app/public/.DS_Store b/app/public/.DS_Store new file mode 100644 index 0000000..55bd2c4 Binary files /dev/null and b/app/public/.DS_Store differ diff --git a/app/public/images/.DS_Store b/app/public/images/.DS_Store new file mode 100644 index 0000000..10edb8a Binary files /dev/null and b/app/public/images/.DS_Store differ diff --git a/fixtures/models.js b/fixtures/models.js index 3bf44f0..bdb3763 100644 --- a/fixtures/models.js +++ b/fixtures/models.js @@ -167,7 +167,7 @@ module.exports = { name: 'ContentType', type: 'contentTypes', jsonapi: { - delete: 'admin', + delete: 'admin', get: 'public', patch: 'admin', post: 'admin' @@ -292,7 +292,7 @@ module.exports = { post: 'admin' }, mockProperties: () => { - return { + return { _id: ObjectId(), apiVersion: '99', authScope: ['foo','bar'], @@ -301,7 +301,6 @@ module.exports = { itemStorageEnabled: true, host: 'sourcehost.example.com', itemDataObjectsFromPagePathTemplate: 'things.${contentTypePluralCamelName}', - itemsGetUrlTemplate: 'https://${host}/test-path/${contentTypePluralCamelName}?foo=bar&access_token=${accessToken}&limit=${limit}&offset=${offset}', itemsLimit: 72, logoGlyphPath: '/source/logoGlyphPath.svg', name: 'Super Source', @@ -320,12 +319,31 @@ module.exports = { logoGlyphPath: String, name: { type: String, required: true }, passportStrategy: String, - itemsGetUrlTemplate: String, itemDataObjectsFromPagePathTemplate: { type: String, default: 'data' }, slug: String, totalItemsAvailableFromPagePathTemplate: String } }, + sourceContentType: { + name: 'SourceContentType', + type: 'sourceContentTypes', + jsonapi: { + get: 'public' + }, + mockProperties: () => { + return { + _id: ObjectId(), + contentType: ObjectId(), + itemsGetUrlTemplate: 'https://${sourceHost}/foo/${contentTypePluralCamelName}?bar=wiz&access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}', + source: ObjectId() + }; + }, + schemaProperties: { + contentType: { ref: 'ContentType', required: true }, + itemsGetUrlTemplate: { type: String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}' }, + source: { ref: 'Source', required: true } + } + }, storage: { name: 'Storage', type: 'storages', @@ -406,7 +424,7 @@ module.exports = { sourceToken: String, sourceUser: String, user: { ref: 'User' } - }, + } }, userStorageAuth: { name: 'UserStorageAuth', diff --git a/package.json b/package.json index 812422d..51c1603 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "debug": "^2.6.0", "emoji-strip": "^1.0.0", "es6-template-strings": "^2.0.1", - "eslint": "^3.17.1", + "eslint": "^4.5.0", "express": "~4.15.0", "express-session": "^1.14.1", "flat": "^2.0.1", @@ -22,7 +22,7 @@ "grunt-contrib-clean": "^1.0.0", "grunt-contrib-symlink": "^1.0.0", "grunt-contrib-watch": "^1.0.0", - "grunt-eslint": "^19.0.0", + "grunt-eslint": "^20.1.0", "grunt-force-task": "^2.0.0", "grunt-jsdoc": "^2.1.0", "grunt-mocha-test": "^0.13.2", @@ -61,6 +61,7 @@ "sanitize-filename": "^1.6.1", "sinon": "^1.17.6", "socket.io": "^1.3.2", + "stack-trace": "0.0.10", "supertest": "^3.0.0", "underscore": "^1.8.3", "url": "^0.11.0", @@ -85,6 +86,7 @@ "devDependencies": { "grunt-deploy-files": "1.0.5", "grunt-env": "^0.4.4", + "grunt-eslint": "^19.0.0", "grunt-hoist": "1.0.4" } -} \ No newline at end of file +} diff --git a/tests/controllers/item/getResource.js b/tests/controllers/item/getResource.js index da426a6..39247d6 100644 --- a/tests/controllers/item/getResource.js +++ b/tests/controllers/item/getResource.js @@ -11,19 +11,19 @@ describe('itemController.getResource method', function() { assertions.function.callbacks.error(controller.getResource, [{ when: 'no url parameter provided', params: [undefined], - error: 'Parameter url undefined or null', + error: 'Parameter url undefined or null' }, { when: 'url parameter not string', params: [3], - error: 'Parameter url is not a string', + error: 'Parameter url is not a string' }, { when: 'url parameter not a valid URL', params: ['asdf'], - error: 'Parameter url is not a properly formatted string', + error: 'Parameter url is not a properly formatted string' }, { when: 'url parameter has unsupported extension', params: ['http://example.com/foo.xyz'], - error: 'Parameter url indicates unsupported media type', + error: 'Parameter url indicates unsupported media type' }, { when: 'url parameter indicates non-existent resource', params: [wh.jsonUrl], diff --git a/tests/controllers/item/itemsGetUrl.js b/tests/controllers/item/itemsGetUrl.js index 8707d49..f5e9b37 100644 --- a/tests/controllers/item/itemsGetUrl.js +++ b/tests/controllers/item/itemsGetUrl.js @@ -7,8 +7,8 @@ var wh = require('app/lib/warehouse'); describe('itemController.itemsGetUrl method', function() { assertions.function.returnsResult('controller.itemsGetUrl', controller.itemsGetUrl, [{ - when: 'provided source (with itemsGetUrlTemplate property), contentType, userSourceAuth, and pagination as parameters', - params: [wh.one('source'), wh.one('contentType'), wh.one('userSourceAuth'), wh.pagination()], + when: 'provided source (with itemsGetUrlTemplate property), sourceContentType, userSourceAuth, and pagination as parameters', + params: [wh.one('source'), wh.one('sourceContentType'), wh.one('userSourceAuth'), wh.pagination()], result: function(itemsGetUrl, done) { assert.equal(itemsGetUrl, templateCompiler(this.params[0].itemsGetUrlTemplate, { host: this.params[0].host, @@ -20,8 +20,8 @@ describe('itemController.itemsGetUrl method', function() { done(); } }, { - when: 'provided source (without itemsGetUrlTemplate property), contentType, userSourceAuth, and pagination as parameters', - params: [wh.one('source'), wh.one('contentType'), wh.one('userSourceAuth'), wh.pagination()], + when: 'provided source (without itemsGetUrlTemplate property), sourceContentType, userSourceAuth, and pagination as parameters', + params: [wh.one('source'), wh.one('sourceContentType'), wh.one('userSourceAuth'), wh.pagination()], result: function(itemsGetUrl, done) { assert.equal(itemsGetUrl, templateCompiler('https://${host}/${contentTypePluralCamelName}?access_token=${accessToken}&limit=${limit}&offset=${offset}', { host: this.params[0].host, @@ -43,27 +43,27 @@ describe('itemController.itemsGetUrl method', function() { assertions.function.throws.error('controller.itemsGetUrl', controller.itemsGetUrl, [{ when: 'no source parameter provided', - params: [undefined, wh.one('contentType'), wh.one('userSourceAuth'), wh.pagination()], + params: [undefined, wh.one('sourceContentType'), wh.one('userSourceAuth'), wh.pagination()], error: 'Parameter source undefined or null' }, { when: 'source parameter has no host property', - params: [{}, wh.one('contentType'), wh.one('userSourceAuth'), wh.pagination()], + params: [{}, wh.one('sourceContentType'), wh.one('userSourceAuth'), wh.pagination()], error: 'Parameter source has no host property' }, { - when: 'no contentType parameter provided', + when: 'no sourceContentType parameter provided', params: [wh.one('source'), undefined, wh.one('userSourceAuth'), wh.pagination()], - error: 'Parameter contentType undefined or null' + error: 'Parameter sourceContentType undefined or null' }, { - when: 'contentType parameter has no name property', + when: 'sourceContentType parameter has no name property', params: [wh.one('source'), {}, wh.one('userSourceAuth'), wh.pagination()], - error: 'Parameter contentType has no name property' + error: 'Parameter sourceContentType has no name property' }, { when: 'no userSourceAuth parameter provided', - params: [wh.one('source'), wh.one('contentType'), undefined, wh.pagination()], + params: [wh.one('source'), wh.one('sourceContentType'), undefined, wh.pagination()], error: 'Parameter userSourceAuth undefined or null' }, { when: 'userSourceAuth parameter has no sourceToken property', - params: [wh.one('source'), wh.one('contentType'), {}, wh.pagination()], + params: [wh.one('source'), wh.one('sourceContentType'), {}, wh.pagination()], error: 'Parameter userSourceAuth has no sourceToken property' }]); }); \ No newline at end of file diff --git a/tests/controllers/item/persistItemDataObject.js b/tests/controllers/item/persistItemDataObject.js index 40d5908..d948478 100644 --- a/tests/controllers/item/persistItemDataObject.js +++ b/tests/controllers/item/persistItemDataObject.js @@ -12,11 +12,11 @@ describe('itemController.persistItemDataObject method', function() { assertions.function.callbacks.error(controller.persistItemDataObject, [{ when: 'no itemDataObject parameter provided', params: [undefined, wh.itemRelationships()], - error: 'Parameter itemDataObject undefined or null', + error: 'Parameter itemDataObject undefined or null' }, { when: 'no relationships parameter provided', params: [wh.itemDataObject(), undefined], - error: 'Parameter relationships undefined or null', + error: 'Parameter relationships undefined or null' }, { when: 'itemDataObject parameter has no id property', params: [wh.itemDataObject(), wh.itemRelationships()],