Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.0] Remove types from Esqueue (#32146) #32230

Merged
merged 1 commit into from
Feb 28, 2019
Merged

[7.0] Remove types from Esqueue (#32146) #32230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions x-pack/plugins/reporting/common/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
* you may not use this file except in compliance with the Elastic License.
*/

export const QUEUE_DOCTYPE = 'esqueue';

export const JOB_COMPLETION_NOTIFICATIONS_SESSION_KEY =
'xpack.reporting.jobCompletionNotifications';

Expand Down
2 changes: 0 additions & 2 deletions x-pack/plugins/reporting/server/lib/create_queue.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

import { Esqueue } from './esqueue';
import { createWorkersFactory } from './create_workers';
import { QUEUE_DOCTYPE } from '../../common/constants';
import { oncePerServer } from './once_per_server';
import { createTaggedLogger } from './create_tagged_logger';

Expand All @@ -20,7 +19,6 @@ function createQueueFn(server) {

const logger = createTaggedLogger(server, ['reporting', 'esqueue']);
const queueOptions = {
doctype: QUEUE_DOCTYPE,
interval: queueConfig.indexInterval,
timeout: queueConfig.timeout,
dateSeparator: dateSeparator,
Expand Down
4 changes: 0 additions & 4 deletions x-pack/plugins/reporting/server/lib/esqueue/__tests__/fixtures/elasticsearch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ ClientMock.prototype.index = function (params = {}) {
const shardCount = 2;
return Promise.resolve({
_index: params.index || 'index',
_type: params.type || constants.DEFAULT_SETTING_DOCTYPE,
_id: params.id || uniqueId('testDoc'),
_seq_no: 1,
_primary_term: 1,
Expand Down Expand Up @@ -52,7 +51,6 @@ ClientMock.prototype.get = function (params = {}, source = {}) {

return Promise.resolve({
_index: params.index || 'index',
_type: params.type || constants.DEFAULT_SETTING_DOCTYPE,
_id: params.id || 'AVRPRLnlp7Ur1SZXfT-T',
_seq_no: params._seq_no || 1,
_primary_term: params._primary_term || 1,
Expand All @@ -65,7 +63,6 @@ ClientMock.prototype.search = function (params = {}, count = 5, source = {}) {
const hits = times(count, () => {
return {
_index: params.index || 'index',
_type: params.type || constants.DEFAULT_SETTING_DOCTYPE,
_id: uniqueId('documentId'),
_seq_no: random(1, 5),
_primar_term: random(1, 5),
Expand Down Expand Up @@ -97,7 +94,6 @@ ClientMock.prototype.update = function (params = {}) {
const shardCount = 2;
return Promise.resolve({
_index: params.index || 'index',
_type: params.type || constants.DEFAULT_SETTING_DOCTYPE,
_id: params.id || uniqueId('testDoc'),
_seq_no: params.if_seq_no + 1 || 2,
_primary_term: params.if_primary_term + 1 || 2,
Expand Down
32 changes: 4 additions & 28 deletions x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/create_index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,8 @@ describe('Create Index', function () {
.then((exists) => expect(exists).to.be(true));
});

it('should create the index with type mappings and default settings', function () {
it('should create the index with mappings and default settings', function () {
const indexName = 'test-index';
const docType = constants.DEFAULT_SETTING_DOCTYPE;
const settings = constants.DEFAULT_SETTING_INDEX_SETTINGS;
const result = createIndex(client, indexName);

Expand All @@ -44,42 +43,20 @@ describe('Create Index', function () {
expect(payload.body).to.have.property('settings');
expect(payload.body.settings).to.eql(settings);
expect(payload.body).to.have.property('mappings');
expect(payload.body.mappings).to.have.property(docType);
expect(payload.body.mappings[docType]).to.have.property('properties');
});
});

it('should accept a custom doctype', function () {
const indexName = 'test-index';
const docType = 'my_type';
const settings = constants.DEFAULT_SETTING_INDEX_SETTINGS;
const result = createIndex(client, indexName, docType);

return result
.then(function () {
const payload = createSpy.getCall(0).args[0];
sinon.assert.callCount(createSpy, 1);
expect(payload).to.have.property('index', indexName);
expect(payload).to.have.property('body');
expect(payload.body).to.have.property('settings');
expect(payload.body.settings).to.eql(settings);
expect(payload.body).to.have.property('mappings');
expect(payload.body.mappings).to.have.property(docType);
expect(payload.body.mappings[docType]).to.have.property('properties');
expect(payload.body.mappings).to.have.property('properties');
});
});

it('should create the index with custom settings', function () {
const indexName = 'test-index';
const docType = constants.DEFAULT_SETTING_DOCTYPE;
const settings = {
...constants.DEFAULT_SETTING_INDEX_SETTINGS,
auto_expand_replicas: false,
number_of_shards: 3000,
number_of_replicas: 1,
format: '3000',
};
const result = createIndex(client, indexName, docType, settings);
const result = createIndex(client, indexName, settings);

return result
.then(function () {
Expand All @@ -90,8 +67,7 @@ describe('Create Index', function () {
expect(payload.body).to.have.property('settings');
expect(payload.body.settings).to.eql(settings);
expect(payload.body).to.have.property('mappings');
expect(payload.body.mappings).to.have.property(docType);
expect(payload.body.mappings[docType]).to.have.property('properties');
expect(payload.body.mappings).to.have.property('properties');
});
});
});
Expand Down
3 changes: 0 additions & 3 deletions x-pack/plugins/reporting/server/lib/esqueue/__tests__/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ describe('Esqueue class', function () {
const job = queue.addJob(jobType, payload);
const options = job.getProp('options');
expect(options).to.have.property('timeout', constants.DEFAULT_SETTING_TIMEOUT);
expect(options).to.have.property('doctype', constants.DEFAULT_SETTING_DOCTYPE);
});

it('should pass queue index settings', function () {
Expand Down Expand Up @@ -149,14 +148,12 @@ describe('Esqueue class', function () {
it('should pass worker options', function () {
const workerOptions = {
size: 12,
doctype: 'tests'
};

queue = new Esqueue('esqueue', { client });
const worker = queue.registerWorker('type', noop, workerOptions);
const options = worker.getProp('options');
expect(options.size).to.equal(workerOptions.size);
expect(options.doctype).to.equal(workerOptions.doctype);
});
});

Expand Down
8 changes: 0 additions & 8 deletions x-pack/plugins/reporting/server/lib/esqueue/__tests__/job.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ describe('Job Class', function () {
const args = createIndexMock.getCall(0).args;
expect(args[0]).to.equal(client);
expect(args[1]).to.equal(index);
expect(args[2]).to.equal(constants.DEFAULT_SETTING_DOCTYPE);
});
});

Expand All @@ -92,7 +91,6 @@ describe('Job Class', function () {
return job.ready.then(() => {
const indexArgs = validateDoc(client.index);
expect(indexArgs).to.have.property('index', index);
expect(indexArgs).to.have.property('type', constants.DEFAULT_SETTING_DOCTYPE);
expect(indexArgs).to.have.property('body');
expect(indexArgs.body).to.have.property('payload', payload);
});
Expand All @@ -103,7 +101,6 @@ describe('Job Class', function () {
return job.ready.then(() => {
const indexArgs = validateDoc(client.index);
expect(indexArgs).to.have.property('index', index);
expect(indexArgs).to.have.property('type', constants.DEFAULT_SETTING_DOCTYPE);
expect(indexArgs).to.have.property('body');
expect(indexArgs.body).to.have.property('jobtype', type);
});
Expand Down Expand Up @@ -134,7 +131,6 @@ describe('Job Class', function () {
try {
expect(jobDoc).to.have.property('id');
expect(jobDoc).to.have.property('index');
expect(jobDoc).to.have.property('type');
expect(jobDoc).to.have.property('_seq_no');
expect(jobDoc).to.have.property('_primary_term');
done();
Expand Down Expand Up @@ -351,7 +347,6 @@ describe('Job Class', function () {
const args = createIndexMock.getCall(0).args;
expect(args[0]).to.equal(newClient);
expect(args[1]).to.equal(index);
expect(args[2]).to.equal(constants.DEFAULT_SETTING_DOCTYPE);
});
});

Expand All @@ -362,7 +357,6 @@ describe('Job Class', function () {

const newDoc = newClient.index.getCall(0).args[0];
expect(newDoc).to.have.property('index', index);
expect(newDoc).to.have.property('type', constants.DEFAULT_SETTING_DOCTYPE);
expect(newDoc).to.have.property('body');
expect(newDoc.body).to.have.property('payload', payload);
});
Expand All @@ -382,7 +376,6 @@ describe('Job Class', function () {
.then((doc) => {
const jobDoc = job.document; // document should be resolved
expect(doc).to.have.property('index', index);
expect(doc).to.have.property('type', jobDoc.type);
expect(doc).to.have.property('id', jobDoc.id);
expect(doc).to.have.property('_seq_no', jobDoc._seq_no);
expect(doc).to.have.property('_primary_term', jobDoc._primary_term);
Expand Down Expand Up @@ -433,7 +426,6 @@ describe('Job Class', function () {

const doc = job.toJSON();
expect(doc).to.have.property('index', index);
expect(doc).to.have.property('type', constants.DEFAULT_SETTING_DOCTYPE);
expect(doc).to.have.property('jobtype', type);
expect(doc).to.have.property('created_by', defaultCreatedBy);
expect(doc).to.have.property('timeout', options.timeout);
Expand Down
25 changes: 0 additions & 25 deletions x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,6 @@ describe('Worker class', function () {
expect(worker).to.have.property('jobtype', jobtype);
expect(worker).to.have.property('workerFn', workerFn);
expect(worker).to.have.property('checkSize');
expect(worker).to.have.property('doctype');
});

it('should have a unique ID', function () {
Expand Down Expand Up @@ -330,24 +329,6 @@ describe('Worker class', function () {
});
});


describe('query parameters', function () {
beforeEach(() => {
searchStub = sinon.stub(mockQueue.client, 'search').callsFake(() => Promise.resolve({ hits: { hits: [] } }));
});

it('should query by default doctype', function () {
const params = getSearchParams();
expect(params).to.have.property('type', constants.DEFAULT_SETTING_DOCTYPE);
});

it('should query by custom doctype', function () {
const doctype = 'custom_test';
const params = getSearchParams('type', { doctype });
expect(params).to.have.property('type', doctype);
});
});

describe('query body', function () {
const conditionPath = 'query.bool.filter.bool';
const jobtype = 'test_jobtype';
Expand Down Expand Up @@ -449,7 +430,6 @@ describe('Worker class', function () {
worker._claimJob(job);
const query = updateSpy.firstCall.args[0];
expect(query).to.have.property('index', job._index);
expect(query).to.have.property('type', job._type);
expect(query).to.have.property('id', job._id);
expect(query).to.have.property('if_seq_no', job._seq_no);
expect(query).to.have.property('if_primary_term', job._primary_term);
Expand Down Expand Up @@ -522,7 +502,6 @@ describe('Worker class', function () {
describe('find a pending job to claim', function () {
const getMockJobs = (status = 'pending') => ([{
_index: 'myIndex',
_type: 'test',
_id: 12345,
_seq_no: 3,
_primary_term: 3,
Expand Down Expand Up @@ -580,7 +559,6 @@ describe('Worker class', function () {
return worker._claimPendingJobs(getMockJobs())
.then(claimedJob => {
expect(claimedJob._index).to.be('myIndex');
expect(claimedJob._type).to.be('test');
expect(claimedJob._source.jobtype).to.be('jobtype');
expect(claimedJob._source.status).to.be('processing');
expect(claimedJob.test).to.be('cool');
Expand Down Expand Up @@ -613,7 +591,6 @@ describe('Worker class', function () {
worker._failJob(job);
const query = updateSpy.firstCall.args[0];
expect(query).to.have.property('index', job._index);
expect(query).to.have.property('type', job._type);
expect(query).to.have.property('id', job._id);
expect(query).to.have.property('if_seq_no', job._seq_no);
expect(query).to.have.property('if_primary_term', job._primary_term);
Expand Down Expand Up @@ -735,7 +712,6 @@ describe('Worker class', function () {
sinon.assert.calledOnce(updateSpy);
const query = updateSpy.firstCall.args[0];
expect(query).to.have.property('index', job._index);
expect(query).to.have.property('type', job._type);
expect(query).to.have.property('id', job._id);
expect(query).to.have.property('if_seq_no', job._seq_no);
expect(query).to.have.property('if_primary_term', job._primary_term);
Expand Down Expand Up @@ -776,7 +752,6 @@ describe('Worker class', function () {
expect(workerJob).to.have.property('job');
expect(workerJob.job).to.have.property('id');
expect(workerJob.job).to.have.property('index');
expect(workerJob.job).to.have.property('type');

expect(workerJob).to.have.property('output');
expect(workerJob.output).to.have.property('content');
Expand Down
1 change: 0 additions & 1 deletion x-pack/plugins/reporting/server/lib/esqueue/constants/default_settings.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ export const defaultSettings = {
DEFAULT_SETTING_TIMEOUT: 10000,
DEFAULT_SETTING_DATE_SEPARATOR: '-',
DEFAULT_SETTING_INTERVAL: 'week',
DEFAULT_SETTING_DOCTYPE: 'esqueue',
DEFAULT_SETTING_INDEX_SETTINGS: {
number_of_shards: 1,
auto_expand_replicas: '0-1',
Expand Down
9 changes: 2 additions & 7 deletions x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,18 +66,14 @@ const schema = {
}
};

export function createIndex(client, indexName,
doctype = constants.DEFAULT_SETTING_DOCTYPE,
indexSettings = { }) {
export function createIndex(client, indexName, indexSettings = {}) {
const body = {
settings: {
...constants.DEFAULT_SETTING_INDEX_SETTINGS,
...indexSettings
},
mappings: {
[doctype]: {
properties: schema
}
properties: schema
}
};

Expand All @@ -88,7 +84,6 @@ export function createIndex(client, indexName,
if (!exists) {
return client.indices.create({
index: indexName,
include_type_name: true,
body: body
})
.then(() => true)
Expand Down
6 changes: 2 additions & 4 deletions x-pack/plugins/reporting/server/lib/esqueue/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ export class Esqueue extends EventEmitter {
this.settings = {
interval: constants.DEFAULT_SETTING_INTERVAL,
timeout: constants.DEFAULT_SETTING_TIMEOUT,
doctype: constants.DEFAULT_SETTING_DOCTYPE,
dateSeparator: constants.DEFAULT_SETTING_DATE_SEPARATOR,
...omit(options, [ 'client' ])
};
Expand All @@ -44,20 +43,19 @@ export class Esqueue extends EventEmitter {
});
}

addJob(type, payload, opts = {}) {
addJob(jobtype, payload, opts = {}) {
const timestamp = indexTimestamp(this.settings.interval, this.settings.dateSeparator);
const index = `${this.index}-${timestamp}`;
const defaults = {
timeout: this.settings.timeout,
};

const options = Object.assign(defaults, opts, {
doctype: this.settings.doctype,
indexSettings: this.settings.indexSettings,
logger: this._logger
});

return new Job(this, index, type, payload, options);
return new Job(this, index, jobtype, payload, options);
}

registerWorker(type, workerFn, opts) {
Expand Down
Loading