Skip to content

Commit

Permalink
add tests for zdt-v2-compat
Browse files Browse the repository at this point in the history
  • Loading branch information
pgayvallet committed May 1, 2023
1 parent 5441475 commit 7eb03fa
Show file tree
Hide file tree
Showing 14 changed files with 547 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* Side Public License, v 1.
*/

import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server';
import { SavedObjectsModelVersion, SavedObjectMigrationFn } from '@kbn/core-saved-objects-server';
import { createType } from '../test_utils';
import { type KibanaMigratorTestKitParams } from '../kibana_migrator_test_kit';

Expand All @@ -29,6 +29,8 @@ export const dummyModelVersion: SavedObjectsModelVersion = {
},
};

export const dummyMigration: SavedObjectMigrationFn = (doc) => doc;

export const getFooType = () => {
return createType({
name: 'foo',
Expand Down Expand Up @@ -130,3 +132,18 @@ export const getExcludedType = () => {
},
});
};

export const getLegacyType = () => {
return createType({
name: 'legacy',
mappings: {
properties: {
someField: { type: 'text' },
},
},
migrations: {
'7.0.0': dummyMigration,
'7.5.0': dummyMigration,
},
});
};
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@ import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server
import '../jest_matchers';
import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit';
import { delay, parseLogFile } from '../test_utils';
import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures';
import {
getBaseMigratorParams,
getSampleAType,
getSampleBType,
} from '../fixtures/zdt_base.fixtures';

export const logFilePath = Path.join(__dirname, 'basic_document_migration.test.log');

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@ import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server
import '../jest_matchers';
import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit';
import { delay, parseLogFile } from '../test_utils';
import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures';
import {
getBaseMigratorParams,
getSampleAType,
getSampleBType,
} from '../fixtures/zdt_base.fixtures';

export const logFilePath = Path.join(__dirname, 'conversion_failures.test.log');

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import '../jest_matchers';
import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit';
import { delay, parseLogFile } from '../test_utils';
import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures';
import { getBaseMigratorParams, getFooType, getBarType } from '../fixtures/zdt_base.fixtures';

export const logFilePath = Path.join(__dirname, 'create_index.test.log');

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import {
getFooType,
getBarType,
dummyModelVersion,
} from './base.fixtures';
} from '../fixtures/zdt_base.fixtures';

export const logFilePath = Path.join(__dirname, 'document_cleanup.test.log');

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ module.exports = {
// see https://github.com/elastic/kibana/pull/130255/
preset: '@kbn/test/jest_integration',
rootDir: '../../../../../../..',
roots: ['<rootDir>/src/core/server/integration_tests/saved_objects/migrations/zero_downtime'],
roots: ['<rootDir>/src/core/server/integration_tests/saved_objects/migrations/zdt_1'],
// must override to match all test given there is no `integration_tests` subfolder
testMatch: ['**/*.test.{js,mjs,ts,tsx}'],
};
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,12 @@ import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-h
import '../jest_matchers';
import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit';
import { delay, parseLogFile } from '../test_utils';
import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures';
import {
getBaseMigratorParams,
getFooType,
getBarType,
dummyModelVersion,
} from '../fixtures/zdt_base.fixtures';

export const logFilePath = Path.join(__dirname, 'mapping_version_conflict.test.log');

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-h
import '../jest_matchers';
import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit';
import { delay, parseLogFile } from '../test_utils';
import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures';
import { getBaseMigratorParams, getFooType, getBarType } from '../fixtures/zdt_base.fixtures';

export const logFilePath = Path.join(__dirname, 'rerun_same_version.test.log');

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
getSampleAType,
getSampleBType,
dummyModelVersion,
} from './base.fixtures';
} from '../fixtures/zdt_base.fixtures';

export const logFilePath = Path.join(__dirname, 'standard_workflow.test.log');

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,12 @@ import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-h
import '../jest_matchers';
import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit';
import { delay, parseLogFile } from '../test_utils';
import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures';
import {
getBaseMigratorParams,
getFooType,
getBarType,
dummyModelVersion,
} from '../fixtures/zdt_base.fixtures';

export const logFilePath = Path.join(__dirname, 'update_mappings.test.log');

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import Path from 'path';
import fs from 'fs/promises';
import { range, sortBy } from 'lodash';
import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server';
import '../jest_matchers';
import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit';
import { delay, parseLogFile } from '../test_utils';
import {
getBaseMigratorParams,
getSampleAType,
getLegacyType,
} from '../fixtures/zdt_base.fixtures';

export const logFilePath = Path.join(__dirname, 'basic_document_migration.test.log');

describe('ZDT with v2 compat - basic document migration', () => {
let esServer: TestElasticsearchUtils['es'];

const startElasticsearch = async () => {
const { startES } = createTestServers({
adjustTimeout: (t: number) => jest.setTimeout(t),
settings: {
es: {
license: 'basic',
},
},
});
return await startES();
};

beforeAll(async () => {
await fs.unlink(logFilePath).catch(() => {});
esServer = await startElasticsearch();
});

afterAll(async () => {
await esServer?.stop();
await delay(10);
});

const createBaseline = async () => {
const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({
...getBaseMigratorParams(),
types: [getSampleAType(), getLegacyType()],
});
await runMigrations();

const sampleAObjs = range(5).map<SavedObjectsBulkCreateObject>((number) => ({
id: `a-${number}`,
type: 'sample_a',
attributes: {
keyword: `a_${number}`,
boolean: true,
},
}));

await savedObjectsRepository.bulkCreate(sampleAObjs);

const legacyTypeObjs = range(5).map<SavedObjectsBulkCreateObject>((number) => ({
id: `legacy-${number}`,
type: 'legacy',
attributes: {
someField: `legacy ${number}`,
},
}));

await savedObjectsRepository.bulkCreate(legacyTypeObjs);
};

it('migrates the documents', async () => {
await createBaseline();

const typeA = getSampleAType();
const legacyType = getLegacyType();

// typeA -> we add a new field and bump the model version by one with a migration

typeA.mappings.properties = {
...typeA.mappings.properties,
someAddedField: { type: 'keyword' },
};

typeA.modelVersions = {
...typeA.modelVersions,
'2': {
modelChange: {
type: 'expansion',
transformation: {
up: (doc) => {
return {
document: {
...doc,
attributes: {
...doc.attributes,
someAddedField: `${doc.attributes.keyword}-mig`,
},
},
};
},
down: jest.fn(),
},
addedMappings: {
someAddedField: { type: 'keyword' },
},
},
},
};

// legacyType -> we add a new migration

legacyType.mappings.properties = {
...legacyType.mappings.properties,
newField: { type: 'text' },
};

legacyType.migrations = {
...legacyType.migrations,
'8.0.0': (document) => {
return {
...document,
attributes: {
...document.attributes,
newField: `populated ${document.id}`,
},
};
},
};

const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({
...getBaseMigratorParams(),
logFilePath,
types: [typeA, legacyType],
});

await runMigrations();

const indices = await client.indices.get({ index: '.kibana*' });
expect(Object.keys(indices)).toEqual(['.kibana_1']);

const index = indices['.kibana_1'];
const mappings = index.mappings ?? {};
const mappingMeta = mappings._meta ?? {};

expect(mappings.properties).toEqual(
expect.objectContaining({
sample_a: typeA.mappings,
legacy: legacyType.mappings,
})
);

expect(mappingMeta.docVersions).toEqual({
sample_a: '10.2.0',
legacy: '8.0.0',
});

const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ type: 'sample_a' });
const { saved_objects: legacyDocs } = await savedObjectsRepository.find({ type: 'legacy' });

expect(sampleADocs).toHaveLength(5);
expect(legacyDocs).toHaveLength(5);

const sampleAData = sortBy(sampleADocs, 'id').map((object) => ({
id: object.id,
type: object.type,
attributes: object.attributes,
}));

expect(sampleAData).toEqual([
{
id: 'a-0',
type: 'sample_a',
attributes: { boolean: true, keyword: 'a_0', someAddedField: 'a_0-mig' },
},
{
id: 'a-1',
type: 'sample_a',
attributes: { boolean: true, keyword: 'a_1', someAddedField: 'a_1-mig' },
},
{
id: 'a-2',
type: 'sample_a',
attributes: { boolean: true, keyword: 'a_2', someAddedField: 'a_2-mig' },
},
{
id: 'a-3',
type: 'sample_a',
attributes: { boolean: true, keyword: 'a_3', someAddedField: 'a_3-mig' },
},
{
id: 'a-4',
type: 'sample_a',
attributes: { boolean: true, keyword: 'a_4', someAddedField: 'a_4-mig' },
},
]);

const sampleBData = sortBy(legacyDocs, 'id').map((object) => ({
id: object.id,
type: object.type,
attributes: object.attributes,
}));

expect(sampleBData).toEqual([
{
id: 'legacy-0',
type: 'legacy',
attributes: { someField: `legacy 0`, newField: `populated legacy-0` },
},
{
id: 'legacy-1',
type: 'legacy',
attributes: { someField: `legacy 1`, newField: `populated legacy-1` },
},
{
id: 'legacy-2',
type: 'legacy',
attributes: { someField: `legacy 2`, newField: `populated legacy-2` },
},
{
id: 'legacy-3',
type: 'legacy',
attributes: { someField: `legacy 3`, newField: `populated legacy-3` },
},
{
id: 'legacy-4',
type: 'legacy',
attributes: { someField: `legacy 4`, newField: `populated legacy-4` },
},
]);

const records = await parseLogFile(logFilePath);
expect(records).toContainLogEntry('Starting to process 10 documents');
expect(records).toContainLogEntry('Migration completed');
});
});
Loading

0 comments on commit 7eb03fa

Please sign in to comment.