Skip to content

Commit

Permalink
Generalise FTR and integration tests to support multiple SO indices
Browse files Browse the repository at this point in the history
  • Loading branch information
@gsoldevila
gsoldevila committed Apr 13, 2023
1 parent 0c154cb commit 984561b
Show file tree
Hide file tree
Showing 20 changed files with 942 additions and 42 deletions.
6 changes: 5 additions & 1 deletion packages/core/saved-objects/core-saved-objects-server/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,11 @@ export type {
SavedObjectsExportablePredicate,
} from './src/saved_objects_management';
export type { SavedObjectStatusMeta } from './src/saved_objects_status';
export { MAIN_SAVED_OBJECT_INDEX } from './src/saved_objects_index_pattern';
export {
MAIN_SAVED_OBJECT_INDEX,
TASK_MANAGER_SAVED_OBJECT_INDEX,
SavedObjectsIndexPatterns,
} from './src/saved_objects_index_pattern';
export type {
SavedObjectsType,
SavedObjectTypeExcludeFromUpgradeFilterHook,
Expand Down
9 changes: 9 additions & 0 deletions packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,13 @@
* Side Public License, v 1.
*/

/**
* Collect and centralize the names of the different saved object indices.
* Note that all of them start with the '.kibana' prefix.
* There are multiple places in the code that these indices have the form .kibana*.
* However, beware that there are some system indices that have the same prefix
* but are NOT used to store saved objects, e.g.: .kibana_security_session_1
*/
export const MAIN_SAVED_OBJECT_INDEX = '.kibana';
export const TASK_MANAGER_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_task_manager`;
export const SavedObjectsIndexPatterns = [MAIN_SAVED_OBJECT_INDEX, TASK_MANAGER_SAVED_OBJECT_INDEX];
9 changes: 2 additions & 7 deletions src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@

import path from 'path';
import { unlink } from 'fs/promises';
import { REPO_ROOT } from '@kbn/repo-info';
import { Env } from '@kbn/config';
import { getEnvOptions } from '@kbn/config-mocks';
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import type { InternalCoreStart } from '@kbn/core-lifecycle-server-internal';
import { Root } from '@kbn/core-root-server-internal';
Expand All @@ -19,8 +16,8 @@ import {
createRootWithCorePlugins,
type TestElasticsearchUtils,
} from '@kbn/core-test-helpers-kbn-server';
import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';

const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version;
const logFilePath = path.join(__dirname, '7.7.2_xpack_100k.log');

async function removeLogFile() {
Expand Down Expand Up @@ -105,8 +102,6 @@ describe('migration from 7.7.2-xpack with 100k objects', () => {
await new Promise((resolve) => setTimeout(resolve, 10000));
};

const migratedIndex = `.kibana_${kibanaVersion}_001`;

beforeAll(async () => {
await removeLogFile();
await startServers({
Expand All @@ -121,7 +116,7 @@ describe('migration from 7.7.2-xpack with 100k objects', () => {

it('copies all the document of the previous index to the new one', async () => {
const migratedIndexResponse = await esClient.count({
index: migratedIndex,
index: SavedObjectsIndexPatterns,
});
const oldIndexResponse = await esClient.count({
index: '.kibana_1',
Expand Down
27 changes: 18 additions & 9 deletions src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import Hapi from '@hapi/hapi';
import h2o2 from '@hapi/h2o2';
import { URL } from 'url';
import type { SavedObject } from '@kbn/core-saved-objects-server';
import { SavedObject, SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
import type { ISavedObjectsRepository } from '@kbn/core-saved-objects-api-server';
import type { InternalCoreSetup, InternalCoreStart } from '@kbn/core-lifecycle-server-internal';
import { Root } from '@kbn/core-root-server-internal';
Expand All @@ -18,6 +18,7 @@ import {
createTestServers,
type TestElasticsearchUtils,
} from '@kbn/core-test-helpers-kbn-server';
import { kibanaPackageJson as pkg } from '@kbn/repo-info';
import {
declareGetRoute,
declareDeleteRoute,
Expand All @@ -30,6 +31,7 @@ import {
declarePostUpdateByQueryRoute,
declarePassthroughRoute,
setProxyInterrupt,
allCombinationsPermutations,
} from './repository_with_proxy_utils';

let esServer: TestElasticsearchUtils;
Expand Down Expand Up @@ -98,17 +100,24 @@ describe('404s from proxies', () => {
await hapiServer.register(h2o2);
// register specific routes to modify the response and a catch-all to relay the request/response as-is

declareGetRoute(hapiServer, esHostname, esPort);
declareDeleteRoute(hapiServer, esHostname, esPort);
declarePostUpdateRoute(hapiServer, esHostname, esPort);
allCombinationsPermutations(
SavedObjectsIndexPatterns.map((indexPattern) => `${indexPattern}_${pkg.version}`)
)
.map((indices) => indices.join(','))
.forEach((kbnIndexPath) => {
declareGetRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declareDeleteRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declarePostUpdateRoute(hapiServer, esHostname, esPort, kbnIndexPath);

declareGetSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declarePostSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declarePostPitRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort, kbnIndexPath);
});

declareGetSearchRoute(hapiServer, esHostname, esPort);
declarePostSearchRoute(hapiServer, esHostname, esPort);
// register index-agnostic routes
declarePostBulkRoute(hapiServer, esHostname, esPort);
declarePostMgetRoute(hapiServer, esHostname, esPort);
declarePostPitRoute(hapiServer, esHostname, esPort);
declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort);

declarePassthroughRoute(hapiServer, esHostname, esPort);

await hapiServer.start();
Expand Down
67 changes: 57 additions & 10 deletions src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
*/
import Hapi from '@hapi/hapi';
import { IncomingMessage } from 'http';
import { kibanaPackageJson as pkg } from '@kbn/repo-info';

// proxy setup
const defaultProxyOptions = (hostname: string, port: string) => ({
Expand Down Expand Up @@ -52,10 +51,13 @@ const proxyOnResponseHandler = async (res: IncomingMessage, h: Hapi.ResponseTool
.code(404);
};

const kbnIndex = `.kibana_${pkg.version}`;

// GET /.kibana_8.0.0/_doc/{type*} route (repository.get calls)
export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declareGetRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'GET',
path: `/${kbnIndex}/_doc/{type*}`,
Expand All @@ -70,7 +72,12 @@ export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port:
},
});
// DELETE /.kibana_8.0.0/_doc/{type*} route (repository.delete calls)
export const declareDeleteRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declareDeleteRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'DELETE',
path: `/${kbnIndex}/_doc/{_id*}`,
Expand Down Expand Up @@ -133,7 +140,12 @@ export const declarePostMgetRoute = (hapiServer: Hapi.Server, hostname: string,
},
});
// GET _search route
export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declareGetSearchRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'GET',
path: `/${kbnIndex}/_search`,
Expand All @@ -149,7 +161,12 @@ export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string,
},
});
// POST _search route (`find` calls)
export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declarePostSearchRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'POST',
path: `/${kbnIndex}/_search`,
Expand All @@ -168,7 +185,12 @@ export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string
},
});
// POST _update
export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declarePostUpdateRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'POST',
path: `/${kbnIndex}/_update/{_id*}`,
Expand All @@ -187,7 +209,12 @@ export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string
},
});
// POST _pit
export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declarePostPitRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'POST',
path: `/${kbnIndex}/_pit`,
Expand All @@ -209,7 +236,8 @@ export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, p
export const declarePostUpdateByQueryRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'POST',
Expand Down Expand Up @@ -244,3 +272,22 @@ export const declarePassthroughRoute = (hapiServer: Hapi.Server, hostname: strin
},
},
});

export function allCombinationsPermutations<T>(collection: T[]): T[][] {
const recur = (subcollection: T[], size: number): T[][] => {
if (size <= 0) {
return [[]];
}
const permutations: T[][] = [];
subcollection.forEach((value, index, array) => {
array = array.slice();
array.splice(index, 1);
recur(array, size - 1).forEach((permutation) => {
permutation.unshift(value);
permutations.push(permutation);
});
});
return permutations;
};
return collection.map((_, n) => recur(collection, n + 1)).flat();
}
103 changes: 103 additions & 0 deletions test/api_integration/fixtures/es_archiver/saved_objects/delete_unknown_types/mappings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,109 @@
"url": "c7f66a0df8b1b52f17c28c4adb111105",
"usage-counters": "8cc260bdceffec4ffc3ad165c97dc1b4",
"visualization": "f819cf6636b75c9e76ba733a0c6ef355"
},
"indexTypesMap": {
".kibana_task_manager": [
"task"
],
".kibana": [
"action_task_params",
"action",
"alert",
"api_key_pending_invalidation",
"apm-indices",
"apm-server-schema",
"apm-service-group",
"apm-telemetry",
"app_search_telemetry",
"application_usage_daily",
"application_usage_totals",
"canvas-element",
"canvas-workpad-template",
"canvas-workpad",
"cases-comments",
"cases-configure",
"cases-connector-mappings",
"cases-telemetry",
"cases-user-actions",
"cases",
"config-global",
"config",
"connector_token",
"core-usage-stats",
"csp-rule-template",
"dashboard",
"endpoint:user-artifact-manifest",
"endpoint:user-artifact",
"enterprise_search_telemetry",
"epm-packages-assets",
"epm-packages",
"event_loop_delays_daily",
"exception-list-agnostic",
"exception-list",
"file-upload-usage-collection-telemetry",
"file",
"fileShare",
"fleet-fleet-server-host",
"fleet-message-signing-keys",
"fleet-preconfiguration-deletion-record",
"fleet-proxy",
"graph-workspace",
"guided-onboarding-guide-state",
"guided-onboarding-plugin-state",
"index-pattern",
"infrastructure-monitoring-log-view",
"infrastructure-ui-source",
"ingest_manager_settings",
"ingest-agent-policies",
"ingest-download-sources",
"ingest-outputs",
"ingest-package-policies",
"inventory-view",
"kql-telemetry",
"legacy-url-alias",
"lens-ui-telemetry",
"lens",
"map",
"metrics-explorer-view",
"ml-job",
"ml-module",
"ml-trained-model",
"monitoring-telemetry",
"osquery-manager-usage-metric",
"osquery-pack-asset",
"osquery-pack",
"osquery-saved-query",
"query",
"rules-settings",
"sample-data-telemetry",
"search-session",
"search-telemetry",
"search",
"security-rule",
"security-solution-signals-migration",
"siem-detection-engine-rule-actions",
"siem-ui-timeline-note",
"siem-ui-timeline-pinned-event",
"siem-ui-timeline",
"slo",
"space",
"spaces-usage-stats",
"synthetics-monitor",
"synthetics-param",
"synthetics-privates-locations",
"tag",
"telemetry",
"ui-metric",
"upgrade-assistant-ml-upgrade-operation",
"upgrade-assistant-reindex-operation",
"uptime-dynamic-settings",
"uptime-synthetics-api-key",
"url",
"usage-counters",
"visualization",
"workplace_search_telemetry"
]
}
},
"dynamic": "strict",
Expand Down
Loading

0 comments on commit 984561b

Please sign in to comment.