Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add write and mWrite #1302

Merged
merged 17 commits into from
May 28, 2019
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions features/kuzzle.feature
Original file line number Diff line number Diff line change
@@ -1,4 +1,28 @@
Feature: Kuzzle functional tests
Scenario: Bulk mWrite
When I create a collection "kuzzle-test-index":"kuzzle-collection-test"
When I use bulk:mWrite action with
"""
{
"documents": [
{ "body": { "name": "Maedhros" } },
{ "body": { "name": "Maglor" } },
{ "body": { "name": "Celegorm" } },
{ "body": { "name": "Caranthis" } },
{ "body": { "name": "Curufin" } },
{ "body": { "name": "Amrod" } },
{ "body": { "name": "Amras" } }
]
}
"""
Then I count 7 documents
And The documents does not have kuzzle metadata

Scenario: Bulk write
When I use bulk:write action with '{ "name": "Feanor", "_kuzzle_info": { "author": "Tolkien" } }'
Then I count 1 documents
And The documents have the following kuzzle metadata '{ "author": "Tolkien" }'

Scenario: Create a collection
When I create a collection "kuzzle-test-index":"my-collection1"
Then The mapping properties field of "kuzzle-test-index":"my-collection1" is "the default value"
Expand Down
33 changes: 33 additions & 0 deletions features/step_definitions/bulk.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
const
should = require('should'),
{
Then,
When
Expand Down Expand Up @@ -116,3 +117,35 @@ When(/^I do a global bulk import$/, function (callback) {
});
});

When('I use bulk:mWrite action with', function (bodyRaw) {
const body = JSON.parse(bodyRaw);

return this.api.bulkMWrite(this.index, this.collection, body);
});

When('I use bulk:write action with {string}', function (bodyRaw) {
const body = JSON.parse(bodyRaw);

return this.api.bulkWrite(this.index, this.collection, body);
});


Then('The documents does not have kuzzle metadata', function () {
return this.api.search({}, this.index, this.collection, { size: 100 })
.then(({ result }) => {
for (const hit of result.hits) {
should(hit._source._kuzzle_info).be.undefined();
}
});
});

Then('The documents have the following kuzzle metadata {string}', function (metadatRaw) {
const metadata = JSON.parse(metadatRaw);

return this.api.search({}, this.index, this.collection, { size: 100 })
.then(({ result }) => {
for (const hit of result.hits) {
should(hit._source._kuzzle_info).match(metadata);
}
});
});
26 changes: 26 additions & 0 deletions features/support/api/apiBase.js
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,32 @@ class ApiBase {
return this.send(msg);
}

bulkMWrite (index, collection, body) {
const
msg = {
controller: 'bulk',
collection: collection || this.world.fakeCollection,
index: index || this.world.fakeIndex,
action: 'mWrite',
body
};

return this.send(msg);
}

bulkWrite (index, collection, body) {
const
msg = {
controller: 'bulk',
collection: collection || this.world.fakeCollection,
index: index || this.world.fakeIndex,
action: 'write',
body
};

return this.send(msg);
}

collectionExists (index, collection) {
return this.send({
index,
Expand Down
20 changes: 20 additions & 0 deletions features/support/api/http.js
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,26 @@ class HttpApi {
return this.callApi(options);
}

bulkMWrite (index, collection, body) {
const options = {
url: this.apiPath(this.util.getIndex(index) + '/' + this.util.getCollection(collection) + '/_mWrite'),
method: 'POST',
body
};

return this.callApi(options);
}

bulkWrite (index, collection, body) {
const options = {
url: this.apiPath(this.util.getIndex(index) + '/' + this.util.getCollection(collection) + '/_write'),
method: 'POST',
body
};

return this.callApi(options);
}

/**
* @param options
* @return {Promise.<IncomingMessage>}
Expand Down
73 changes: 70 additions & 3 deletions lib/api/controllers/bulkController.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,27 @@

const
BaseController = require('./controller'),
{errors: {PartialError}} = require('kuzzle-common-objects'),
{assertHasBody, assertBodyHasAttribute} = require('../../util/requestAssertions');
{ errors: { PartialError } } = require('kuzzle-common-objects'),
{
assertHasBody,
assertBodyHasAttribute,
assertHasIndexAndCollection,
assertBodyAttributeType,
assertIdStartsNotUnderscore
} = require('../../util/requestAssertions');

/**
* @class BulkController
* @param {Kuzzle} kuzzle
*/
class BulkController extends BaseController {
constructor (kuzzle) {
super(kuzzle, ['import']);
super(kuzzle, [
'import',
'write',
'mWrite'
]);

this.engine = kuzzle.services.list.storageEngine;
}

Expand All @@ -55,6 +66,62 @@ class BulkController extends BaseController {
return response;
});
}

/**
* Write a document without adding metadata or performing data validation.
* @param {Request} request
* @returns {Promise<Object>}
*/
write(request) {
assertHasBody(request);
assertHasIndexAndCollection(request);
assertIdStartsNotUnderscore(request);

const notify = this.tryGetBoolean(request, 'args.notify');

return this.engine.createOrReplace(request, false)
.then(response => {
this.kuzzle.indexCache.add(request.input.resource.index, request.input.resource.collection);

if (notify && response.created) {
this.kuzzle.notifier.notifyDocumentCreate(request, response);
}
else if (notify) {
Aschen marked this conversation as resolved.
Show resolved Hide resolved
this.kuzzle.notifier.notifyDocumentReplace(request);
}

return response;
});
}

/**
* Write several documents without adding metadata or performing data validation.
*
* @param {Request} request
* @returns {Promise<Object>}
*/
mWrite(request) {
assertHasBody(request);
assertBodyHasAttribute(request, 'documents');
assertBodyAttributeType(request, 'documents', 'array');
assertHasIndexAndCollection(request);

const notify = this.tryGetBoolean(request, 'args.notify');

return this.engine.mcreateOrReplace(request, false)
.then(response => {
if (response.error.length > 0) {
request.setError(new PartialError('Some document creations failed', response.error));
}

if (notify) {
this.kuzzle.notifier.notifyDocumentMChanges(request, response.result, true);
}

return { hits: response.result, total: response.result.length };
});
}

}

module.exports = BulkController;
8 changes: 4 additions & 4 deletions lib/api/controllers/collectionController.js
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ class CollectionController extends BaseController {
getMapping(request) {
assertHasIndexAndCollection(request);

this.ensureBooleanFlag(request, 'input.args.includeKuzzleMeta');
const includeKuzzleMeta = this.tryGetBoolean(request, 'args.includeKuzzleMeta');

return this.engine.getMapping(request);
return this.engine.getMapping(request, includeKuzzleMeta);
}

/**
Expand Down Expand Up @@ -354,12 +354,12 @@ const formatSpecification = (index, collection, validation) => ({
*/
function createSpecificationList (request) {
const { index, collection } = request.input.resource;

if (index && collection) {
const specification = formatSpecification(index, collection, request.input.body);
return Bluebird.resolve([ specification ]);
}

const specifications = [];
_.forEach(request.input.body, (collections, _index) => {
_.forEach(collections, (validation, _collection) => {
Expand Down
9 changes: 6 additions & 3 deletions lib/api/controllers/controller.js
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,16 @@ class BaseController {
}

/**
* Ensure an input args is a boolean.
* Get a boolean param from request input
* For HTTP, flag presence mean true value
*
* @param {Request} request
* @param {string} flagPath
*/
ensureBooleanFlag (request, flagPath) {
tryGetBoolean (request, flagPath) {
const
flagName = flagPath.split('.').slice(-1),
flagValue = _.get(request, flagPath);
flagValue = _.get(request, `input.${flagPath}`);

// In HTTP, booleans are flags: if it's in the querystring, it's set, whatever
// its value.
Expand All @@ -82,8 +82,11 @@ class BaseController {
const booleanValue = flagValue !== undefined ? true : false;

_.set(request, flagPath, booleanValue);

return booleanValue;
}

return Boolean(flagValue);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is not ensureBooleanFlag anymore, since you're now retrieving the value

another name must now be found, for instance tryGetBoolean?

also, the jsdoc is now outdated

}
}

Expand Down
2 changes: 1 addition & 1 deletion lib/api/core/plugins/pluginContext.js
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ class PluginContext {
const Kuzzle = require('../../kuzzle');

this.accessors = {};
this.config = kuzzle.config;
this.config = JSON.parse(JSON.stringify(kuzzle.config));
this.constructors = deprecateProperties({
RequestContext,
RequestInput,
Expand Down
7 changes: 5 additions & 2 deletions lib/config/httpRoutes.js
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ module.exports = [
{verb: 'post', url: '/credentials/:strategy/_me/_create', controller: 'auth', action: 'createMyCredentials'},
{verb: 'post', url: '/credentials/:strategy/_me/_validate', controller: 'auth', action: 'validateMyCredentials'},

/* DEPRECATED - will be removed in v2 */
/* DEPRECATED - will be removed in v2 */
{verb: 'post', url: '/_validateSpecifications', controller: 'collection', action: 'validateSpecifications'},

{verb: 'post', url: '/:index/:collection/_validateSpecifications', controller: 'collection', action: 'validateSpecifications'},
Expand All @@ -163,6 +163,9 @@ module.exports = [
{verb: 'post', url: '/:index/_bulk', controller: 'bulk', action: 'import'},
{verb: 'post', url: '/:index/:collection/_bulk', controller: 'bulk', action: 'import'},

{verb: 'post', url: '/:index/:collection/_mWrite', controller: 'bulk', action: 'mWrite'},
{verb: 'post', url: '/:index/:collection/_write', controller: 'bulk', action: 'write'},

/* DEPRECATED - will be removed in v2 */
{verb: 'post', url: '/_getStats', controller: 'server', action: 'getStats'},

Expand Down Expand Up @@ -296,7 +299,7 @@ module.exports = [

/* DEPRECATED - will be removed in v2 */
{verb: 'put', url: '/_specifications', controller: 'collection', action: 'updateSpecifications'},

{verb: 'put', url: '/:index/:collection/_specifications', controller: 'collection', action: 'updateSpecifications'},

{verb: 'put', url: '/:index/:collection/:_id', controller: 'document', action: 'createOrReplace'},
Expand Down
Loading