From 5a8150b4ee2ae24c700d59bfcb4d11c1276c949f Mon Sep 17 00:00:00 2001 From: Tre' Seymour Date: Wed, 3 Feb 2021 16:19:34 -0700 Subject: [PATCH 01/15] [QA] Create Saved Objects FTR Svc Add svc, integrate into one test, add exported saved objects, add empty kibana call, and update ftr schema...add data directory for the new svc. --- .../lib/config/schema.ts | 9 +++ test/functional/apps/discover/_discover.ts | 7 +- .../discover/exported.ndjson | 4 + test/functional/services/index.ts | 2 + .../saved_objects/fetch_saved_objects.ts | 81 +++++++++++++++++++ .../saved_objects/import_saved_objects.ts | 39 +++++++++ .../services/saved_objects/index.ts | 21 +++++ .../services/saved_objects/utils.ts | 23 ++++++ 8 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 test/functional/fixtures/exported_saved_objects/discover/exported.ndjson create mode 100644 test/functional/services/saved_objects/fetch_saved_objects.ts create mode 100644 test/functional/services/saved_objects/import_saved_objects.ts create mode 100644 test/functional/services/saved_objects/index.ts create mode 100644 test/functional/services/saved_objects/utils.ts diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 4fd28678d2653..5df58b55d40d9 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -213,6 +213,15 @@ export const schema = Joi.object() }) .default(), + // settings for the saved objects svc + savedObjects: Joi.object() + .keys({ + directory: Joi.string().default( + defaultRelativeToConfigPath('fixtures/exported_saved_objects') + ), + }) + .default(), + // settings for the kibanaServer.uiSettings module uiSettings: Joi.object() .keys({ diff --git a/test/functional/apps/discover/_discover.ts b/test/functional/apps/discover/_discover.ts index 9323c9e2fe70b..bc6a81aab3e8b 100644 --- a/test/functional/apps/discover/_discover.ts +++ b/test/functional/apps/discover/_discover.ts @@ -19,7 +19,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const queryBar = getService('queryBar'); const inspector = getService('inspector'); const elasticChart = getService('elasticChart'); + const savedObjects = getService('savedObjects'); + const supertest = getService('supertest'); const PageObjects = getPageObjects(['common', 'discover', 'header', 'timePicker']); + const defaultSettings = { defaultIndex: 'logstash-*', }; @@ -27,7 +30,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('discover test', function describeIndexTests() { before(async function () { log.debug('load kibana index with default index pattern'); - await esArchiver.load('discover'); + + await esArchiver.load('empty_kibana'); + await savedObjects.import('discover')(log)(supertest); // and load a set of makelogs data await esArchiver.loadIfNeeded('logstash_functional'); diff --git a/test/functional/fixtures/exported_saved_objects/discover/exported.ndjson b/test/functional/fixtures/exported_saved_objects/discover/exported.ndjson new file mode 100644 index 0000000000000..274aacc57482d --- /dev/null +++ b/test/functional/fixtures/exported_saved_objects/discover/exported.ndjson @@ -0,0 +1,4 @@ +{"attributes":{"fieldAttrs":"{\"referer\":{\"customLabel\":\"Referer custom\"}}","fields":"[{\"name\":\"@message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@message\"}}},{\"name\":\"@tags\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@tags\"}}},{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"agent\"}}},{\"name\":\"bytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"extension\"}}},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"headings\"}}},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"host\"}}},{\"name\":\"id\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"index\"}}},{\"name\":\"ip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"links\"}}},{\"name\":\"machine.os\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"machine.os\"}}},{\"name\":\"machine.ram\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"esTypes\":[\"double\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"nestedField.child\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"nested\":{\"path\":\"nestedField\"}}},{\"name\":\"phpmemory\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:section\"}}},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:tag\"}}},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:description\"}}},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image\"}}},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:height\"}}},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:width\"}}},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:site_name\"}}},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:title\"}}},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:type\"}}},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:url\"}}},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:card\"}}},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:description\"}}},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:image\"}}},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:site\"}}},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:title\"}}},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.url\"}}},{\"name\":\"request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"request\"}}},{\"name\":\"response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"response\"}}},{\"name\":\"spaces\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"spaces\"}}},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"url\"}}},{\"name\":\"utc_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"xss\"}}}]","timeFieldName":"@timestamp","title":"logstash-*"},"coreMigrationVersion":"8.0.0","id":"logstash-*","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","version":"WzQsMl0="} + +{"attributes":{"columns":["_source"],"description":"A Saved Search Description","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"filter\":[],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"pre712":true,"sort":[["@timestamp","desc"]],"title":"A Saved Search","version":1},"coreMigrationVersion":"8.0.0","id":"ab12e3c0-f231-11e6-9486-733b1ac9221a","migrationVersion":{"search":"7.12.0"},"references":[{"id":"logstash-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","version":"WzUsMl0="} + diff --git a/test/functional/services/index.ts b/test/functional/services/index.ts index 94d7b71c640c3..2c8ef8e985018 100644 --- a/test/functional/services/index.ts +++ b/test/functional/services/index.ts @@ -46,6 +46,7 @@ import { ListingTableProvider } from './listing_table'; import { SavedQueryManagementComponentProvider } from './saved_query_management_component'; import { KibanaSupertestProvider } from './supertest'; import { MenuToggleProvider } from './menu_toggle'; +import { SavedObjectsProvider } from './saved_objects'; export const services = { ...commonServiceProviders, @@ -82,4 +83,5 @@ export const services = { supertest: KibanaSupertestProvider, managementMenu: ManagementMenuProvider, MenuToggle: MenuToggleProvider, + savedObjects: SavedObjectsProvider, }; diff --git a/test/functional/services/saved_objects/fetch_saved_objects.ts b/test/functional/services/saved_objects/fetch_saved_objects.ts new file mode 100644 index 0000000000000..b70dbcba3e970 --- /dev/null +++ b/test/functional/services/saved_objects/fetch_saved_objects.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { writeFileSync } from 'fs'; +// @ts-ignore +import { join } from 'path'; +// @ts-ignore +import { ToolingLog } from '@kbn/dev-utils'; +import { SuperTest } from 'supertest'; +import { mkDir, finalDirAndFile, ndjsonToObj, mark } from './utils'; +// @ts-ignore +import * as Either from '../../../../src/dev/code_coverage/ingest_coverage/either'; + +const encoding = 'utf8'; + +const writeUtf8 = { flag: 'w', encoding }; + +const appendUtf8 = { flag: 'a', encoding }; + +const defaultTypes = ['index-pattern', 'search', 'visualization', 'dashboard']; + +const isEmptyEither = (x: string) => (x === '' ? Either.left(x) : Either.right(x)); + +const prokEither = (resp: any) => isEmptyEither(resp.text).map(ndjsonToObj); + +export const exportSavedObjects = (types = defaultTypes, excludeExportDetails = true) => async ( + log: ToolingLog, + supertest: SuperTest +) => + await supertest + .post('/api/saved_objects/_export') + .set('kbn-xsrf', 'anything') + .send({ + type: types, + excludeExportDetails, + }) + .expect(200) + .expect('Content-Type', /json/) + .then(prokEither); + +const writeOrAppend = (x: number) => (x === 0 ? Either.left(x) : Either.right(x)); + +const flush = (dest: string) => (savedObj: any, i: number) => { + const writeToFile = writeFileSync.bind(null, dest); + const withTrailingNewLines: string = `${JSON.stringify(savedObj)}\n\n`; + const writeCleaned = writeToFile.bind(null, withTrailingNewLines); + + writeOrAppend(i).fold( + // @ts-ignore + () => writeCleaned(writeUtf8), + // @ts-ignore + () => writeCleaned(appendUtf8) + ); +}; + +export const flushSavedObjects = (dest: string) => (log: ToolingLog) => (payloadEither: any) => + payloadEither.fold( + () => log.debug(`${mark} Empty resp.text for:\n\t ${dest}`), + (payload: any) => { + log.verbose(`${mark} payload: \n\t${JSON.stringify(payload, null, 2)}`); + const flushToDestination = flush(dest); + [...payload].forEach(flushToDestination); + log.debug(`${mark} Exported saved objects to destination: \n\t${dest}`); + } + ); + +export const fetchSavedObjects = (dataDir: string) => (appName: string) => ( + log: ToolingLog +) => async (supertest: SuperTest) => { + const joined = join(dataDir, appName); + const [destDir, destFilePath] = finalDirAndFile(joined)(); + + mkDir(destDir); + const exportUsingDefaults = exportSavedObjects(); + await flushSavedObjects(destFilePath)(log)(await exportUsingDefaults(log, supertest)); +}; diff --git a/test/functional/services/saved_objects/import_saved_objects.ts b/test/functional/services/saved_objects/import_saved_objects.ts new file mode 100644 index 0000000000000..8d69a42157319 --- /dev/null +++ b/test/functional/services/saved_objects/import_saved_objects.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { readFileSync } from 'fs'; +import { SuperTest } from 'supertest'; +// @ts-ignore +import { ToolingLog } from '@kbn/dev-utils'; +import { join } from 'path'; +import { finalDirAndFile, mark } from './utils'; + +export const importData = (srcFilePath: string) => async ( + log: ToolingLog, + supertest: SuperTest +) => + await supertest + .post('/api/saved_objects/_import') + .query({ overwrite: true }) + .set('kbn-xsrf', 'anything') + .attach('file', readFileSync(srcFilePath), srcFilePath) + .expect(200) + .then(() => log.debug(`${mark} import successful of ${srcFilePath}`)) + .catch((err: any) => + log.debug(`${mark} caught error - import response: \n\t${JSON.stringify(err, null, 2)}`) + ); + +export const importSavedObjects = (dataDir: string) => (appName: string) => ( + log: ToolingLog +) => async (supertest: SuperTest) => { + const from = join(dataDir, appName); + const [, inputFilePath] = finalDirAndFile(from)(); + + log.debug(`${mark} Importing saved objects from path: \n\t${inputFilePath}`); + importData(inputFilePath)(log, supertest); +}; diff --git a/test/functional/services/saved_objects/index.ts b/test/functional/services/saved_objects/index.ts new file mode 100644 index 0000000000000..24ebb79aed7b9 --- /dev/null +++ b/test/functional/services/saved_objects/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { fetchSavedObjects } from './fetch_saved_objects'; +import { importSavedObjects } from './import_saved_objects'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; + +export const SavedObjectsProvider = ({ getService }: FtrProviderContext) => { + const config = getService('config'); + const dataDir = config.get('savedObjects.directory'); + + return { + fetch: fetchSavedObjects(dataDir), + import: importSavedObjects(dataDir), + }; +}; diff --git a/test/functional/services/saved_objects/utils.ts b/test/functional/services/saved_objects/utils.ts new file mode 100644 index 0000000000000..1c98e0a5e8d33 --- /dev/null +++ b/test/functional/services/saved_objects/utils.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { join } from 'path'; +// @ts-ignore +import shell from 'shelljs'; + +export const finalDirAndFile = (destDir: string) => (filePath = 'exported.ndjson') => { + const destFilePath = join(destDir, filePath); + return [destDir, destFilePath]; +}; + +export const mkDir = (x: string) => shell.mkdir('-p', x); + +// @ts-ignore +export const ndjsonToObj = (x: string) => x.split('\n').map(JSON.parse); + +export const mark = '[SavedObjsSvc]'; From b4e6f4a5cb57e0829bd2b05c7df5c6598ce6a38e Mon Sep 17 00:00:00 2001 From: Tre' Seymour Date: Thu, 11 Feb 2021 13:40:39 -0700 Subject: [PATCH 02/15] [QA] Create Saved Objects FTR Svc Make public api take one argument. --- test/functional/apps/discover/_discover.ts | 3 +-- .../exported_saved_objects/discover/exported.ndjson | 4 ++-- .../services/saved_objects/fetch_saved_objects.ts | 6 +++--- .../services/saved_objects/import_saved_objects.ts | 6 +++--- test/functional/services/saved_objects/index.ts | 6 ++++-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/test/functional/apps/discover/_discover.ts b/test/functional/apps/discover/_discover.ts index bc6a81aab3e8b..8ae98af0b0804 100644 --- a/test/functional/apps/discover/_discover.ts +++ b/test/functional/apps/discover/_discover.ts @@ -20,7 +20,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const inspector = getService('inspector'); const elasticChart = getService('elasticChart'); const savedObjects = getService('savedObjects'); - const supertest = getService('supertest'); const PageObjects = getPageObjects(['common', 'discover', 'header', 'timePicker']); const defaultSettings = { @@ -32,7 +31,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { log.debug('load kibana index with default index pattern'); await esArchiver.load('empty_kibana'); - await savedObjects.import('discover')(log)(supertest); + await savedObjects.import('discover'); // and load a set of makelogs data await esArchiver.loadIfNeeded('logstash_functional'); diff --git a/test/functional/fixtures/exported_saved_objects/discover/exported.ndjson b/test/functional/fixtures/exported_saved_objects/discover/exported.ndjson index 274aacc57482d..578f175642c9d 100644 --- a/test/functional/fixtures/exported_saved_objects/discover/exported.ndjson +++ b/test/functional/fixtures/exported_saved_objects/discover/exported.ndjson @@ -1,4 +1,4 @@ -{"attributes":{"fieldAttrs":"{\"referer\":{\"customLabel\":\"Referer custom\"}}","fields":"[{\"name\":\"@message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@message\"}}},{\"name\":\"@tags\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@tags\"}}},{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"agent\"}}},{\"name\":\"bytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"extension\"}}},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"headings\"}}},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"host\"}}},{\"name\":\"id\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"index\"}}},{\"name\":\"ip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"links\"}}},{\"name\":\"machine.os\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"machine.os\"}}},{\"name\":\"machine.ram\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"esTypes\":[\"double\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"nestedField.child\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"nested\":{\"path\":\"nestedField\"}}},{\"name\":\"phpmemory\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:section\"}}},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:tag\"}}},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:description\"}}},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image\"}}},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:height\"}}},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:width\"}}},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:site_name\"}}},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:title\"}}},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:type\"}}},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:url\"}}},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:card\"}}},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:description\"}}},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:image\"}}},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:site\"}}},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:title\"}}},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.url\"}}},{\"name\":\"request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"request\"}}},{\"name\":\"response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"response\"}}},{\"name\":\"spaces\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"spaces\"}}},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"url\"}}},{\"name\":\"utc_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"xss\"}}}]","timeFieldName":"@timestamp","title":"logstash-*"},"coreMigrationVersion":"8.0.0","id":"logstash-*","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","version":"WzQsMl0="} +{"attributes":{"fieldAttrs":"{\"referer\":{\"customLabel\":\"Referer custom\"}}","fields":"[{\"name\":\"@message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@message\"}}},{\"name\":\"@tags\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@tags\"}}},{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"agent\"}}},{\"name\":\"bytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"extension\"}}},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"headings\"}}},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"host\"}}},{\"name\":\"id\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"index\"}}},{\"name\":\"ip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"links\"}}},{\"name\":\"machine.os\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"machine.os\"}}},{\"name\":\"machine.ram\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"esTypes\":[\"double\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"nestedField.child\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"nested\":{\"path\":\"nestedField\"}}},{\"name\":\"phpmemory\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:section\"}}},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:tag\"}}},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:description\"}}},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image\"}}},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:height\"}}},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:width\"}}},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:site_name\"}}},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:title\"}}},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:type\"}}},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:url\"}}},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:card\"}}},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:description\"}}},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:image\"}}},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:site\"}}},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:title\"}}},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.url\"}}},{\"name\":\"request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"request\"}}},{\"name\":\"response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"response\"}}},{\"name\":\"spaces\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"spaces\"}}},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"url\"}}},{\"name\":\"utc_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"xss\"}}}]","timeFieldName":"@timestamp","title":"logstash-*"},"coreMigrationVersion":"8.0.0","id":"logstash-*","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[-9223372036854776000,2],"type":"index-pattern","version":"WzQsMl0="} -{"attributes":{"columns":["_source"],"description":"A Saved Search Description","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"filter\":[],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"pre712":true,"sort":[["@timestamp","desc"]],"title":"A Saved Search","version":1},"coreMigrationVersion":"8.0.0","id":"ab12e3c0-f231-11e6-9486-733b1ac9221a","migrationVersion":{"search":"7.12.0"},"references":[{"id":"logstash-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","version":"WzUsMl0="} +{"attributes":{"columns":["_source"],"description":"A Saved Search Description","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"filter\":[],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"pre712":true,"sort":[["@timestamp","desc"]],"title":"A Saved Search","version":1},"coreMigrationVersion":"8.0.0","id":"ab12e3c0-f231-11e6-9486-733b1ac9221a","migrationVersion":{"search":"7.12.0"},"references":[{"id":"logstash-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[-9223372036854776000,4],"type":"search","version":"WzUsMl0="} diff --git a/test/functional/services/saved_objects/fetch_saved_objects.ts b/test/functional/services/saved_objects/fetch_saved_objects.ts index b70dbcba3e970..4440112dc6486 100644 --- a/test/functional/services/saved_objects/fetch_saved_objects.ts +++ b/test/functional/services/saved_objects/fetch_saved_objects.ts @@ -69,9 +69,9 @@ export const flushSavedObjects = (dest: string) => (log: ToolingLog) => (payload } ); -export const fetchSavedObjects = (dataDir: string) => (appName: string) => ( - log: ToolingLog -) => async (supertest: SuperTest) => { +export const fetchSavedObjects = (dataDir: string) => (log: ToolingLog) => ( + supertest: SuperTest +) => async (appName: string) => { const joined = join(dataDir, appName); const [destDir, destFilePath] = finalDirAndFile(joined)(); diff --git a/test/functional/services/saved_objects/import_saved_objects.ts b/test/functional/services/saved_objects/import_saved_objects.ts index 8d69a42157319..8549cce8d3aa7 100644 --- a/test/functional/services/saved_objects/import_saved_objects.ts +++ b/test/functional/services/saved_objects/import_saved_objects.ts @@ -28,9 +28,9 @@ export const importData = (srcFilePath: string) => async ( log.debug(`${mark} caught error - import response: \n\t${JSON.stringify(err, null, 2)}`) ); -export const importSavedObjects = (dataDir: string) => (appName: string) => ( - log: ToolingLog -) => async (supertest: SuperTest) => { +export const importSavedObjects = (dataDir: string) => (log: ToolingLog) => ( + supertest: SuperTest +) => async (appName: string) => { const from = join(dataDir, appName); const [, inputFilePath] = finalDirAndFile(from)(); diff --git a/test/functional/services/saved_objects/index.ts b/test/functional/services/saved_objects/index.ts index 24ebb79aed7b9..364dc3e3f51d6 100644 --- a/test/functional/services/saved_objects/index.ts +++ b/test/functional/services/saved_objects/index.ts @@ -12,10 +12,12 @@ import { FtrProviderContext } from '../../../common/ftr_provider_context'; export const SavedObjectsProvider = ({ getService }: FtrProviderContext) => { const config = getService('config'); + const log = getService('log'); + const supertest = getService('supertest'); const dataDir = config.get('savedObjects.directory'); return { - fetch: fetchSavedObjects(dataDir), - import: importSavedObjects(dataDir), + fetch: fetchSavedObjects(dataDir)(log)(supertest), + import: importSavedObjects(dataDir)(log)(supertest), }; }; From 94a0709d4a1794302a87c1d201df016166e1300e Mon Sep 17 00:00:00 2001 From: Tre' Seymour Date: Thu, 11 Feb 2021 13:48:35 -0700 Subject: [PATCH 03/15] [QA] Create Saved Objects FTR Svc Make typescript happier. --- test/functional/services/saved_objects/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/functional/services/saved_objects/index.ts b/test/functional/services/saved_objects/index.ts index 364dc3e3f51d6..2f520898a31e0 100644 --- a/test/functional/services/saved_objects/index.ts +++ b/test/functional/services/saved_objects/index.ts @@ -13,11 +13,14 @@ import { FtrProviderContext } from '../../../common/ftr_provider_context'; export const SavedObjectsProvider = ({ getService }: FtrProviderContext) => { const config = getService('config'); const log = getService('log'); + // @ts-ignore const supertest = getService('supertest'); const dataDir = config.get('savedObjects.directory'); return { + // @ts-ignore fetch: fetchSavedObjects(dataDir)(log)(supertest), + // @ts-ignore import: importSavedObjects(dataDir)(log)(supertest), }; }; From 800c310c95c5308d38b1a7191f48a3a48d9d0c36 Mon Sep 17 00:00:00 2001 From: spalger Date: Tue, 16 Feb 2021 17:08:39 -0700 Subject: [PATCH 04/15] refactor to import/export APIs on kbnClient --- package.json | 1 + .../src/kbn_client/kbn_client.ts | 8 ++ .../kbn_client/kbn_client_import_export.ts | 92 +++++++++++++++ .../src/kbn_client/kbn_client_requester.ts | 2 + packages/kbn-test/src/index.ts | 2 + packages/kbn-test/src/saved_objects_cli.ts | 105 ++++++++++++++++++ scripts/saved_objects.js | 10 ++ .../services/kibana_server/kibana_server.ts | 1 + test/functional/apps/discover/_discover.ts | 3 +- .../exported.ndjson => discover.json} | 0 test/functional/services/index.ts | 2 - .../saved_objects/fetch_saved_objects.ts | 81 -------------- .../saved_objects/import_saved_objects.ts | 39 ------- .../services/saved_objects/index.ts | 26 ----- .../services/saved_objects/utils.ts | 23 ---- yarn.lock | 9 ++ 16 files changed, 231 insertions(+), 173 deletions(-) create mode 100644 packages/kbn-dev-utils/src/kbn_client/kbn_client_import_export.ts create mode 100644 packages/kbn-test/src/saved_objects_cli.ts create mode 100644 scripts/saved_objects.js rename test/functional/fixtures/exported_saved_objects/{discover/exported.ndjson => discover.json} (100%) delete mode 100644 test/functional/services/saved_objects/fetch_saved_objects.ts delete mode 100644 test/functional/services/saved_objects/import_saved_objects.ts delete mode 100644 test/functional/services/saved_objects/index.ts delete mode 100644 test/functional/services/saved_objects/utils.ts diff --git a/package.json b/package.json index cc8ab44c51c96..177eabb39b8b5 100644 --- a/package.json +++ b/package.json @@ -655,6 +655,7 @@ "fetch-mock": "^7.3.9", "file-loader": "^4.2.0", "file-saver": "^1.3.8", + "form-data": "^4.0.0", "formsy-react": "^1.1.5", "geckodriver": "^1.21.0", "glob-watcher": "5.0.3", diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts index 963639d47045b..bc91af47155d2 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts @@ -13,12 +13,14 @@ import { KbnClientPlugins } from './kbn_client_plugins'; import { KbnClientVersion } from './kbn_client_version'; import { KbnClientSavedObjects } from './kbn_client_saved_objects'; import { KbnClientUiSettings, UiSettingValues } from './kbn_client_ui_settings'; +import { KbnClientImportExport } from './kbn_client_import_export'; export interface KbnClientOptions { url: string; certificateAuthorities?: Buffer[]; log: ToolingLog; uiSettingDefaults?: UiSettingValues; + importExportDir?: string; } export class KbnClient { @@ -27,6 +29,7 @@ export class KbnClient { readonly version: KbnClientVersion; readonly savedObjects: KbnClientSavedObjects; readonly uiSettings: KbnClientUiSettings; + readonly importExport: KbnClientImportExport; private readonly requester: KbnClientRequester; private readonly log: ToolingLog; @@ -56,6 +59,11 @@ export class KbnClient { this.version = new KbnClientVersion(this.status); this.savedObjects = new KbnClientSavedObjects(this.log, this.requester); this.uiSettings = new KbnClientUiSettings(this.log, this.requester, this.uiSettingDefaults); + this.importExport = new KbnClientImportExport( + this.log, + this.requester, + options.importExportDir + ); } /** diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_import_export.ts new file mode 100644 index 0000000000000..46e9ba4cc8c2a --- /dev/null +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_import_export.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { inspect } from 'util'; +import Fs from 'fs/promises'; +import Path from 'path'; + +import FormData from 'form-data'; + +import { ToolingLog } from '../tooling_log'; +import { KbnClientRequester, uriencode } from './kbn_client_requester'; + +export class KbnClientImportExport { + constructor( + public readonly log: ToolingLog, + public readonly requester: KbnClientRequester, + public readonly dir?: string + ) {} + + resolvePath(path: string) { + if (!Path.extname(path)) { + path = `${path}.json`; + } + + if (!this.dir && !Path.isAbsolute(path)) { + throw new Error( + '[KbnClientImportExport] unable to resolve relative path to import/export without a configured dir' + ); + } + + return this.dir ? Path.resolve(this.dir, path) : Path.resolve(path); + } + + async import(name: string, options?: { space?: string }) { + const src = this.resolvePath(name); + + const flattenedJson = (await Fs.readFile(src, 'utf-8')) + .split('\n\n') + .map((json) => JSON.stringify(JSON.parse(json))) + .join('\n'); + + const formData = new FormData(); + formData.append('file', flattenedJson); + + // TODO: should we clear out the existing saved objects? + + await this.requester.request({ + method: 'POST', + path: options?.space + ? uriencode`/s/${options.space}/api/saved_objects/_import` + : '/api/saved_objects/_import', + query: { + overwrite: true, + }, + body: formData, + headers: formData.getHeaders(), + }); + + // TODO: we should verify that the import was successfull by inspecting the response + + this.log.debug(`[KbnClientImportExport] import successful of ${src}`); + } + + async export(name: string, options?: { space?: string }) { + const dest = this.resolvePath(name); + + const resp = await this.requester.request({ + method: 'POST', + path: options?.space + ? uriencode`/s/${options.space}/api/saved_objects/_export` + : '/api/saved_objects/_export', + }); + + if (typeof resp.data !== 'string' || !resp.data) { + throw new Error( + `[KbnClientImportExport] unexpected response from export API: ${inspect(resp.data)}` + ); + } + + const objects = resp.data.split('\n').map((line) => JSON.parse(line)); + const fileContents = objects.map((obj) => JSON.stringify(obj, null, 2)).join('\n\n'); + + await Fs.writeFile(dest, fileContents, 'utf-8'); + + this.log.debug(`[KbnClientImportExport] Exported saved objects to destination:\n ${dest}`); + } +} diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts index d940525f57e3c..11ae7bd4acaf2 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts @@ -52,6 +52,7 @@ export interface ReqOptions { method: 'GET' | 'POST' | 'PUT' | 'DELETE'; body?: any; retries?: number; + headers?: Record; } const delay = (ms: number) => @@ -102,6 +103,7 @@ export class KbnClientRequester { data: options.body, params: options.query, headers: { + ...options.headers, 'kbn-xsrf': 'kbn-client', }, httpsAgent: this.httpsAgent, diff --git a/packages/kbn-test/src/index.ts b/packages/kbn-test/src/index.ts index 25a5c6541bf07..ea124553b6f9c 100644 --- a/packages/kbn-test/src/index.ts +++ b/packages/kbn-test/src/index.ts @@ -48,3 +48,5 @@ export { getUrl } from './jest/utils/get_url'; export { runCheckJestConfigsCli } from './jest/run_check_jest_configs_cli'; export { runJest } from './jest/run'; + +export * from './saved_objects_cli'; diff --git a/packages/kbn-test/src/saved_objects_cli.ts b/packages/kbn-test/src/saved_objects_cli.ts new file mode 100644 index 0000000000000..570cacd5ae728 --- /dev/null +++ b/packages/kbn-test/src/saved_objects_cli.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Url from 'url'; + +import { RunWithCommands, createFlagError, Flags, KbnClient } from '@kbn/dev-utils'; + +import { readConfigFile } from './functional_test_runner'; + +function getSinglePositionalArg(flags: Flags) { + const positional = flags._; + if (positional.length < 1) { + throw createFlagError('missing name of export to import'); + } + + if (positional.length > 1) { + throw createFlagError(`extra positional arguments, expected 1, got [${positional}]`); + } + + return positional[0]; +} + +export function runSavedObjectsCli() { + new RunWithCommands({ + description: 'Import/export saved objects from archives, for testing', + globalFlags: { + string: ['config'], + help: ` + --config optional path to an FTR config file that will be parsed and used for defaults + --kibana-url set the url that kibana can be reached at, uses the URL from --config by default, or localhost:5601 + --dir directory that contains exports to be imported, or where exports will be saved, uses the path from --config + by default or failes when not specified. + `, + }, + async extendContext({ log, flags }) { + let config; + if (!flags.config) { + if (typeof flags.config !== 'string') { + throw createFlagError('expected --config to be a string'); + } + + config = await readConfigFile(log, Path.resolve(flags.config)); + } + + let kibanaUrl; + if (flags['kibana-url']) { + if (typeof flags['kibana-url'] !== 'string') { + throw createFlagError('expected --kibana-url to be a string'); + } + + kibanaUrl = flags['kibana-url']; + } else if (config) { + kibanaUrl = Url.format(config.get('servers.kibana')); + } + + let importExportDir; + if (flags.dir) { + if (typeof flags.dir !== 'string') { + throw createFlagError('expected --dir to be a string'); + } + + importExportDir = flags.dir; + } else if (config) { + importExportDir = config.get('savedObjects.directory'); + } + + if (!importExportDir) { + throw createFlagError( + '--config does not include a savedObjects.directory, specify it or include --dir flag' + ); + } + + return { + kbnClient: new KbnClient({ + log, + url: kibanaUrl ?? 'http://localhost:5601', + importExportDir, + }), + }; + }, + }) + .command({ + name: 'import', + usage: 'import ', + description: 'import a saved export to Kibana', + async run({ kbnClient, flags }) { + await kbnClient.importExport.import(getSinglePositionalArg(flags)); + }, + }) + .command({ + name: 'export', + usage: 'export ', + description: 'export saved objects from Kibana to a file', + async run({ kbnClient, flags }) { + await kbnClient.importExport.export(getSinglePositionalArg(flags)); + }, + }) + .execute(); +} diff --git a/scripts/saved_objects.js b/scripts/saved_objects.js new file mode 100644 index 0000000000000..589d4dd19c2bb --- /dev/null +++ b/scripts/saved_objects.js @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +require('../src/setup_node_env'); +require('@kbn/test').runSavedObjectsCli(); diff --git a/test/common/services/kibana_server/kibana_server.ts b/test/common/services/kibana_server/kibana_server.ts index 51f9dc9d00772..03beb19f5ec57 100644 --- a/test/common/services/kibana_server/kibana_server.ts +++ b/test/common/services/kibana_server/kibana_server.ts @@ -22,6 +22,7 @@ export function KibanaServerProvider({ getService }: FtrProviderContext) { url, certificateAuthorities: config.get('servers.kibana.certificateAuthorities'), uiSettingDefaults: defaults, + importExportDir: config.get('savedObjects.directory'), }); if (defaults) { diff --git a/test/functional/apps/discover/_discover.ts b/test/functional/apps/discover/_discover.ts index 8ae98af0b0804..7d4302968f2df 100644 --- a/test/functional/apps/discover/_discover.ts +++ b/test/functional/apps/discover/_discover.ts @@ -19,7 +19,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const queryBar = getService('queryBar'); const inspector = getService('inspector'); const elasticChart = getService('elasticChart'); - const savedObjects = getService('savedObjects'); const PageObjects = getPageObjects(['common', 'discover', 'header', 'timePicker']); const defaultSettings = { @@ -31,7 +30,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { log.debug('load kibana index with default index pattern'); await esArchiver.load('empty_kibana'); - await savedObjects.import('discover'); + await kibanaServer.importExport.import('discover'); // and load a set of makelogs data await esArchiver.loadIfNeeded('logstash_functional'); diff --git a/test/functional/fixtures/exported_saved_objects/discover/exported.ndjson b/test/functional/fixtures/exported_saved_objects/discover.json similarity index 100% rename from test/functional/fixtures/exported_saved_objects/discover/exported.ndjson rename to test/functional/fixtures/exported_saved_objects/discover.json diff --git a/test/functional/services/index.ts b/test/functional/services/index.ts index 2c8ef8e985018..94d7b71c640c3 100644 --- a/test/functional/services/index.ts +++ b/test/functional/services/index.ts @@ -46,7 +46,6 @@ import { ListingTableProvider } from './listing_table'; import { SavedQueryManagementComponentProvider } from './saved_query_management_component'; import { KibanaSupertestProvider } from './supertest'; import { MenuToggleProvider } from './menu_toggle'; -import { SavedObjectsProvider } from './saved_objects'; export const services = { ...commonServiceProviders, @@ -83,5 +82,4 @@ export const services = { supertest: KibanaSupertestProvider, managementMenu: ManagementMenuProvider, MenuToggle: MenuToggleProvider, - savedObjects: SavedObjectsProvider, }; diff --git a/test/functional/services/saved_objects/fetch_saved_objects.ts b/test/functional/services/saved_objects/fetch_saved_objects.ts deleted file mode 100644 index 4440112dc6486..0000000000000 --- a/test/functional/services/saved_objects/fetch_saved_objects.ts +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { writeFileSync } from 'fs'; -// @ts-ignore -import { join } from 'path'; -// @ts-ignore -import { ToolingLog } from '@kbn/dev-utils'; -import { SuperTest } from 'supertest'; -import { mkDir, finalDirAndFile, ndjsonToObj, mark } from './utils'; -// @ts-ignore -import * as Either from '../../../../src/dev/code_coverage/ingest_coverage/either'; - -const encoding = 'utf8'; - -const writeUtf8 = { flag: 'w', encoding }; - -const appendUtf8 = { flag: 'a', encoding }; - -const defaultTypes = ['index-pattern', 'search', 'visualization', 'dashboard']; - -const isEmptyEither = (x: string) => (x === '' ? Either.left(x) : Either.right(x)); - -const prokEither = (resp: any) => isEmptyEither(resp.text).map(ndjsonToObj); - -export const exportSavedObjects = (types = defaultTypes, excludeExportDetails = true) => async ( - log: ToolingLog, - supertest: SuperTest -) => - await supertest - .post('/api/saved_objects/_export') - .set('kbn-xsrf', 'anything') - .send({ - type: types, - excludeExportDetails, - }) - .expect(200) - .expect('Content-Type', /json/) - .then(prokEither); - -const writeOrAppend = (x: number) => (x === 0 ? Either.left(x) : Either.right(x)); - -const flush = (dest: string) => (savedObj: any, i: number) => { - const writeToFile = writeFileSync.bind(null, dest); - const withTrailingNewLines: string = `${JSON.stringify(savedObj)}\n\n`; - const writeCleaned = writeToFile.bind(null, withTrailingNewLines); - - writeOrAppend(i).fold( - // @ts-ignore - () => writeCleaned(writeUtf8), - // @ts-ignore - () => writeCleaned(appendUtf8) - ); -}; - -export const flushSavedObjects = (dest: string) => (log: ToolingLog) => (payloadEither: any) => - payloadEither.fold( - () => log.debug(`${mark} Empty resp.text for:\n\t ${dest}`), - (payload: any) => { - log.verbose(`${mark} payload: \n\t${JSON.stringify(payload, null, 2)}`); - const flushToDestination = flush(dest); - [...payload].forEach(flushToDestination); - log.debug(`${mark} Exported saved objects to destination: \n\t${dest}`); - } - ); - -export const fetchSavedObjects = (dataDir: string) => (log: ToolingLog) => ( - supertest: SuperTest -) => async (appName: string) => { - const joined = join(dataDir, appName); - const [destDir, destFilePath] = finalDirAndFile(joined)(); - - mkDir(destDir); - const exportUsingDefaults = exportSavedObjects(); - await flushSavedObjects(destFilePath)(log)(await exportUsingDefaults(log, supertest)); -}; diff --git a/test/functional/services/saved_objects/import_saved_objects.ts b/test/functional/services/saved_objects/import_saved_objects.ts deleted file mode 100644 index 8549cce8d3aa7..0000000000000 --- a/test/functional/services/saved_objects/import_saved_objects.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { readFileSync } from 'fs'; -import { SuperTest } from 'supertest'; -// @ts-ignore -import { ToolingLog } from '@kbn/dev-utils'; -import { join } from 'path'; -import { finalDirAndFile, mark } from './utils'; - -export const importData = (srcFilePath: string) => async ( - log: ToolingLog, - supertest: SuperTest -) => - await supertest - .post('/api/saved_objects/_import') - .query({ overwrite: true }) - .set('kbn-xsrf', 'anything') - .attach('file', readFileSync(srcFilePath), srcFilePath) - .expect(200) - .then(() => log.debug(`${mark} import successful of ${srcFilePath}`)) - .catch((err: any) => - log.debug(`${mark} caught error - import response: \n\t${JSON.stringify(err, null, 2)}`) - ); - -export const importSavedObjects = (dataDir: string) => (log: ToolingLog) => ( - supertest: SuperTest -) => async (appName: string) => { - const from = join(dataDir, appName); - const [, inputFilePath] = finalDirAndFile(from)(); - - log.debug(`${mark} Importing saved objects from path: \n\t${inputFilePath}`); - importData(inputFilePath)(log, supertest); -}; diff --git a/test/functional/services/saved_objects/index.ts b/test/functional/services/saved_objects/index.ts deleted file mode 100644 index 2f520898a31e0..0000000000000 --- a/test/functional/services/saved_objects/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { fetchSavedObjects } from './fetch_saved_objects'; -import { importSavedObjects } from './import_saved_objects'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; - -export const SavedObjectsProvider = ({ getService }: FtrProviderContext) => { - const config = getService('config'); - const log = getService('log'); - // @ts-ignore - const supertest = getService('supertest'); - const dataDir = config.get('savedObjects.directory'); - - return { - // @ts-ignore - fetch: fetchSavedObjects(dataDir)(log)(supertest), - // @ts-ignore - import: importSavedObjects(dataDir)(log)(supertest), - }; -}; diff --git a/test/functional/services/saved_objects/utils.ts b/test/functional/services/saved_objects/utils.ts deleted file mode 100644 index 1c98e0a5e8d33..0000000000000 --- a/test/functional/services/saved_objects/utils.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { join } from 'path'; -// @ts-ignore -import shell from 'shelljs'; - -export const finalDirAndFile = (destDir: string) => (filePath = 'exported.ndjson') => { - const destFilePath = join(destDir, filePath); - return [destDir, destFilePath]; -}; - -export const mkDir = (x: string) => shell.mkdir('-p', x); - -// @ts-ignore -export const ndjsonToObj = (x: string) => x.split('\n').map(JSON.parse); - -export const mark = '[SavedObjsSvc]'; diff --git a/yarn.lock b/yarn.lock index 1a939e3dddf57..34b1070ea38e3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14928,6 +14928,15 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" From a1e793d7be16bd8ae9abba95fe835328e5874483 Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Feb 2021 12:09:34 -0700 Subject: [PATCH 05/15] move `KbnClient` to @kbn/test, fix CLI, verify successful response --- packages/kbn-dev-utils/src/index.ts | 1 - .../src/actions/empty_kibana_index.ts | 3 +- packages/kbn-es-archiver/src/actions/load.ts | 3 +- .../kbn-es-archiver/src/actions/unload.ts | 3 +- packages/kbn-es-archiver/src/cli.ts | 4 +- packages/kbn-es-archiver/src/es_archiver.ts | 3 +- .../src/lib/indices/kibana_index.ts | 3 +- .../lib/config/schema.ts | 6 +- packages/kbn-test/src/index.ts | 4 +- ...ved_objects_cli.ts => kbn_archiver_cli.ts} | 41 +++++++------ .../src/kbn_client/index.ts | 0 .../src/kbn_client/kbn_client.ts | 3 +- .../kbn_client/kbn_client_import_export.ts | 57 ++++++++++++------- .../src/kbn_client/kbn_client_plugins.ts | 0 .../src/kbn_client/kbn_client_requester.ts | 5 +- .../kbn_client/kbn_client_saved_objects.ts | 2 +- .../src/kbn_client/kbn_client_status.ts | 0 .../src/kbn_client/kbn_client_ui_settings.ts | 2 +- .../src/kbn_client/kbn_client_version.ts | 0 scripts/{saved_objects.js => kbn_archiver.js} | 2 +- .../services/kibana_server/kibana_server.ts | 2 +- test/common/services/security/role.ts | 3 +- .../common/services/security/role_mappings.ts | 3 +- test/common/services/security/user.ts | 3 +- .../discover.json | 0 .../case/server/scripts/sub_cases/index.ts | 3 +- .../common/endpoint/index_data.ts | 2 +- .../kbn_client_with_api_key_support.ts | 2 +- .../endpoint/resolver_generator_script.ts | 3 +- .../scripts/endpoint/trusted_apps/index.ts | 3 +- .../detection_engine_api_integration/utils.ts | 2 +- 31 files changed, 102 insertions(+), 66 deletions(-) rename packages/kbn-test/src/{saved_objects_cli.ts => kbn_archiver_cli.ts} (76%) rename packages/{kbn-dev-utils => kbn-test}/src/kbn_client/index.ts (100%) rename packages/{kbn-dev-utils => kbn-test}/src/kbn_client/kbn_client.ts (98%) rename packages/{kbn-dev-utils => kbn-test}/src/kbn_client/kbn_client_import_export.ts (61%) rename packages/{kbn-dev-utils => kbn-test}/src/kbn_client/kbn_client_plugins.ts (100%) rename packages/{kbn-dev-utils => kbn-test}/src/kbn_client/kbn_client_requester.ts (97%) rename packages/{kbn-dev-utils => kbn-test}/src/kbn_client/kbn_client_saved_objects.ts (98%) rename packages/{kbn-dev-utils => kbn-test}/src/kbn_client/kbn_client_status.ts (100%) rename packages/{kbn-dev-utils => kbn-test}/src/kbn_client/kbn_client_ui_settings.ts (98%) rename packages/{kbn-dev-utils => kbn-test}/src/kbn_client/kbn_client_version.ts (100%) rename scripts/{saved_objects.js => kbn_archiver.js} (90%) rename test/functional/fixtures/{exported_saved_objects => kbn_archiver}/discover.json (100%) diff --git a/packages/kbn-dev-utils/src/index.ts b/packages/kbn-dev-utils/src/index.ts index 66ad4e7be589b..3ac3927d25c05 100644 --- a/packages/kbn-dev-utils/src/index.ts +++ b/packages/kbn-dev-utils/src/index.ts @@ -23,7 +23,6 @@ export { KBN_P12_PATH, KBN_P12_PASSWORD, } from './certs'; -export * from './kbn_client'; export * from './run'; export * from './axios'; export * from './stdio'; diff --git a/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts b/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts index f86865ffa6670..821e3a872820e 100644 --- a/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts +++ b/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts @@ -7,7 +7,8 @@ */ import { Client } from '@elastic/elasticsearch'; -import { ToolingLog, KbnClient } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { migrateKibanaIndex, createStats, cleanKibanaIndices } from '../lib'; diff --git a/packages/kbn-es-archiver/src/actions/load.ts b/packages/kbn-es-archiver/src/actions/load.ts index e1552107eec7c..b42437beab597 100644 --- a/packages/kbn-es-archiver/src/actions/load.ts +++ b/packages/kbn-es-archiver/src/actions/load.ts @@ -9,7 +9,8 @@ import { resolve } from 'path'; import { createReadStream } from 'fs'; import { Readable } from 'stream'; -import { ToolingLog, KbnClient } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { Client } from '@elastic/elasticsearch'; import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils'; diff --git a/packages/kbn-es-archiver/src/actions/unload.ts b/packages/kbn-es-archiver/src/actions/unload.ts index c12fa935f786a..b5f259a1496bb 100644 --- a/packages/kbn-es-archiver/src/actions/unload.ts +++ b/packages/kbn-es-archiver/src/actions/unload.ts @@ -10,7 +10,8 @@ import { resolve } from 'path'; import { createReadStream } from 'fs'; import { Readable, Writable } from 'stream'; import { Client } from '@elastic/elasticsearch'; -import { ToolingLog, KbnClient } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { createPromiseFromStreams } from '@kbn/utils'; import { diff --git a/packages/kbn-es-archiver/src/cli.ts b/packages/kbn-es-archiver/src/cli.ts index 919aff5c3851b..9617457d4573e 100644 --- a/packages/kbn-es-archiver/src/cli.ts +++ b/packages/kbn-es-archiver/src/cli.ts @@ -17,8 +17,8 @@ import Url from 'url'; import readline from 'readline'; import Fs from 'fs'; -import { RunWithCommands, createFlagError, KbnClient, CA_CERT_PATH } from '@kbn/dev-utils'; -import { readConfigFile } from '@kbn/test'; +import { RunWithCommands, createFlagError, CA_CERT_PATH } from '@kbn/dev-utils'; +import { readConfigFile, KbnClient } from '@kbn/test'; import { Client } from '@elastic/elasticsearch'; import { EsArchiver } from './es_archiver'; diff --git a/packages/kbn-es-archiver/src/es_archiver.ts b/packages/kbn-es-archiver/src/es_archiver.ts index b00b9fb8b3f25..68eacb4f3caf2 100644 --- a/packages/kbn-es-archiver/src/es_archiver.ts +++ b/packages/kbn-es-archiver/src/es_archiver.ts @@ -7,7 +7,8 @@ */ import { Client } from '@elastic/elasticsearch'; -import { ToolingLog, KbnClient } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { saveAction, diff --git a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts index 64e5626c94c8b..76b84f40f34c7 100644 --- a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts @@ -9,7 +9,8 @@ import { inspect } from 'util'; import { Client } from '@elastic/elasticsearch'; -import { ToolingLog, KbnClient } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { Stats } from '../stats'; import { deleteIndex } from './delete_index'; diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 5df58b55d40d9..0694bc4ffdb0f 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -214,11 +214,9 @@ export const schema = Joi.object() .default(), // settings for the saved objects svc - savedObjects: Joi.object() + kbnArchiver: Joi.object() .keys({ - directory: Joi.string().default( - defaultRelativeToConfigPath('fixtures/exported_saved_objects') - ), + directory: Joi.string().default(defaultRelativeToConfigPath('fixtures/kbn_archiver')), }) .default(), diff --git a/packages/kbn-test/src/index.ts b/packages/kbn-test/src/index.ts index ea124553b6f9c..919dc8b4477f3 100644 --- a/packages/kbn-test/src/index.ts +++ b/packages/kbn-test/src/index.ts @@ -49,4 +49,6 @@ export { runCheckJestConfigsCli } from './jest/run_check_jest_configs_cli'; export { runJest } from './jest/run'; -export * from './saved_objects_cli'; +export * from './kbn_archiver_cli'; + +export * from './kbn_client'; diff --git a/packages/kbn-test/src/saved_objects_cli.ts b/packages/kbn-test/src/kbn_archiver_cli.ts similarity index 76% rename from packages/kbn-test/src/saved_objects_cli.ts rename to packages/kbn-test/src/kbn_archiver_cli.ts index 570cacd5ae728..9a42308602fe8 100644 --- a/packages/kbn-test/src/saved_objects_cli.ts +++ b/packages/kbn-test/src/kbn_archiver_cli.ts @@ -9,7 +9,8 @@ import Path from 'path'; import Url from 'url'; -import { RunWithCommands, createFlagError, Flags, KbnClient } from '@kbn/dev-utils'; +import { RunWithCommands, createFlagError, Flags } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { readConfigFile } from './functional_test_runner'; @@ -26,21 +27,21 @@ function getSinglePositionalArg(flags: Flags) { return positional[0]; } -export function runSavedObjectsCli() { +export function runKbnArchiverCli() { new RunWithCommands({ description: 'Import/export saved objects from archives, for testing', globalFlags: { string: ['config'], help: ` --config optional path to an FTR config file that will be parsed and used for defaults - --kibana-url set the url that kibana can be reached at, uses the URL from --config by default, or localhost:5601 - --dir directory that contains exports to be imported, or where exports will be saved, uses the path from --config - by default or failes when not specified. + --kibana-url set the url that kibana can be reached at, uses the "servers.kibana" setting from --config by default + --dir directory that contains exports to be imported, or where exports will be saved, uses the "kbnArchiver.directory" + setting from --config by default `, }, async extendContext({ log, flags }) { let config; - if (!flags.config) { + if (flags.config) { if (typeof flags.config !== 'string') { throw createFlagError('expected --config to be a string'); } @@ -59,6 +60,12 @@ export function runSavedObjectsCli() { kibanaUrl = Url.format(config.get('servers.kibana')); } + if (!kibanaUrl) { + throw createFlagError( + 'Either a --config file with `servers.kibana` defined, or a --kibana-url must be passed' + ); + } + let importExportDir; if (flags.dir) { if (typeof flags.dir !== 'string') { @@ -67,38 +74,38 @@ export function runSavedObjectsCli() { importExportDir = flags.dir; } else if (config) { - importExportDir = config.get('savedObjects.directory'); + importExportDir = config.get('kbnArchiver.directory'); } if (!importExportDir) { throw createFlagError( - '--config does not include a savedObjects.directory, specify it or include --dir flag' + '--config does not include a kbnArchiver.directory, specify it or include --dir flag' ); } return { kbnClient: new KbnClient({ log, - url: kibanaUrl ?? 'http://localhost:5601', + url: kibanaUrl, importExportDir, }), }; }, }) .command({ - name: 'import', - usage: 'import ', - description: 'import a saved export to Kibana', + name: 'save', + usage: 'save ', + description: 'export saved objects from Kibana to a file', async run({ kbnClient, flags }) { - await kbnClient.importExport.import(getSinglePositionalArg(flags)); + await kbnClient.importExport.export(getSinglePositionalArg(flags)); }, }) .command({ - name: 'export', - usage: 'export ', - description: 'export saved objects from Kibana to a file', + name: 'load', + usage: 'load ', + description: 'import a saved export to Kibana', async run({ kbnClient, flags }) { - await kbnClient.importExport.export(getSinglePositionalArg(flags)); + await kbnClient.importExport.import(getSinglePositionalArg(flags)); }, }) .execute(); diff --git a/packages/kbn-dev-utils/src/kbn_client/index.ts b/packages/kbn-test/src/kbn_client/index.ts similarity index 100% rename from packages/kbn-dev-utils/src/kbn_client/index.ts rename to packages/kbn-test/src/kbn_client/index.ts diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts b/packages/kbn-test/src/kbn_client/kbn_client.ts similarity index 98% rename from packages/kbn-dev-utils/src/kbn_client/kbn_client.ts rename to packages/kbn-test/src/kbn_client/kbn_client.ts index bc91af47155d2..08092f32bebb1 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client.ts @@ -6,7 +6,8 @@ * Side Public License, v 1. */ -import { ToolingLog } from '../tooling_log'; +import { ToolingLog } from '@kbn/dev-utils'; + import { KbnClientRequester, ReqOptions } from './kbn_client_requester'; import { KbnClientStatus } from './kbn_client_status'; import { KbnClientPlugins } from './kbn_client_plugins'; diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts similarity index 61% rename from packages/kbn-dev-utils/src/kbn_client/kbn_client_import_export.ts rename to packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index 46e9ba4cc8c2a..c2bf775f60136 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -9,10 +9,12 @@ import { inspect } from 'util'; import Fs from 'fs/promises'; import Path from 'path'; +import { Readable } from 'stream'; import FormData from 'form-data'; +import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils'; -import { ToolingLog } from '../tooling_log'; +import type { SavedObjectsImportResponse } from 'src/core/server/saved_objects/import/types'; import { KbnClientRequester, uriencode } from './kbn_client_requester'; export class KbnClientImportExport { @@ -39,31 +41,48 @@ export class KbnClientImportExport { async import(name: string, options?: { space?: string }) { const src = this.resolvePath(name); - const flattenedJson = (await Fs.readFile(src, 'utf-8')) + const objects = (await Fs.readFile(src, 'utf-8')) .split('\n\n') - .map((json) => JSON.stringify(JSON.parse(json))) - .join('\n'); + .filter((line) => !!line) + .map((line) => JSON.parse(line)); + + this.log.debug('importing', objects.length, 'saved objects'); const formData = new FormData(); - formData.append('file', flattenedJson); + formData.append('file', objects.map((obj) => JSON.stringify(obj)).join('\n'), 'import.ndjson'); // TODO: should we clear out the existing saved objects? - await this.requester.request({ - method: 'POST', - path: options?.space - ? uriencode`/s/${options.space}/api/saved_objects/_import` - : '/api/saved_objects/_import', - query: { - overwrite: true, - }, - body: formData, - headers: formData.getHeaders(), - }); - - // TODO: we should verify that the import was successfull by inspecting the response + let resp; + try { + resp = await this.requester.request({ + method: 'POST', + path: options?.space + ? uriencode`/s/${options.space}/api/saved_objects/_import` + : '/api/saved_objects/_import', + query: { + overwrite: true, + }, + body: formData, + headers: formData.getHeaders(), + }); + } catch (error) { + if (!isAxiosResponseError(error)) { + throw error; + } + + throw createFailError( + `[KbnClientImportExport] ${error.response.status} resp: ${inspect(error.response.data)}` + ); + } - this.log.debug(`[KbnClientImportExport] import successful of ${src}`); + if (resp.data.success) { + this.log.debug('[KbnClientImportExport] import success'); + } else { + throw createFailError( + `[KbnClientImportExport] failed to import all saved objects: ${inspect(resp.data)}` + ); + } } async export(name: string, options?: { space?: string }) { diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_plugins.ts b/packages/kbn-test/src/kbn_client/kbn_client_plugins.ts similarity index 100% rename from packages/kbn-dev-utils/src/kbn_client/kbn_client_plugins.ts rename to packages/kbn-test/src/kbn_client/kbn_client_plugins.ts diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts b/packages/kbn-test/src/kbn_client/kbn_client_requester.ts similarity index 97% rename from packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts rename to packages/kbn-test/src/kbn_client/kbn_client_requester.ts index 11ae7bd4acaf2..8bdfe7da35ddc 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_requester.ts @@ -8,10 +8,9 @@ import Url from 'url'; import Https from 'https'; -import Axios, { AxiosResponse } from 'axios'; -import { isAxiosRequestError, isAxiosResponseError } from '../axios'; -import { ToolingLog } from '../tooling_log'; +import Axios, { AxiosResponse } from 'axios'; +import { ToolingLog, isAxiosRequestError, isAxiosResponseError } from '@kbn/dev-utils'; const isConcliftOnGetError = (error: any) => { return ( diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts b/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts similarity index 98% rename from packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts rename to packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts index 9d616d6f50a88..b175e979631ee 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { ToolingLog } from '../tooling_log'; +import { ToolingLog } from '@kbn/dev-utils'; import { KbnClientRequester, uriencode } from './kbn_client_requester'; diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_status.ts b/packages/kbn-test/src/kbn_client/kbn_client_status.ts similarity index 100% rename from packages/kbn-dev-utils/src/kbn_client/kbn_client_status.ts rename to packages/kbn-test/src/kbn_client/kbn_client_status.ts diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_ui_settings.ts b/packages/kbn-test/src/kbn_client/kbn_client_ui_settings.ts similarity index 98% rename from packages/kbn-dev-utils/src/kbn_client/kbn_client_ui_settings.ts rename to packages/kbn-test/src/kbn_client/kbn_client_ui_settings.ts index 75fd7a4c8391e..78155098ef038 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_ui_settings.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_ui_settings.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { ToolingLog } from '../tooling_log'; +import { ToolingLog } from '@kbn/dev-utils'; import { KbnClientRequester, uriencode } from './kbn_client_requester'; diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_version.ts b/packages/kbn-test/src/kbn_client/kbn_client_version.ts similarity index 100% rename from packages/kbn-dev-utils/src/kbn_client/kbn_client_version.ts rename to packages/kbn-test/src/kbn_client/kbn_client_version.ts diff --git a/scripts/saved_objects.js b/scripts/kbn_archiver.js similarity index 90% rename from scripts/saved_objects.js rename to scripts/kbn_archiver.js index 589d4dd19c2bb..b04b86a0d4eed 100644 --- a/scripts/saved_objects.js +++ b/scripts/kbn_archiver.js @@ -7,4 +7,4 @@ */ require('../src/setup_node_env'); -require('@kbn/test').runSavedObjectsCli(); +require('@kbn/test').runKbnArchiverCli(); diff --git a/test/common/services/kibana_server/kibana_server.ts b/test/common/services/kibana_server/kibana_server.ts index 03beb19f5ec57..5b1ea1429873a 100644 --- a/test/common/services/kibana_server/kibana_server.ts +++ b/test/common/services/kibana_server/kibana_server.ts @@ -7,7 +7,7 @@ */ import Url from 'url'; -import { KbnClient } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { FtrProviderContext } from '../../ftr_provider_context'; diff --git a/test/common/services/security/role.ts b/test/common/services/security/role.ts index 2aae5b2282940..420bed027f317 100644 --- a/test/common/services/security/role.ts +++ b/test/common/services/security/role.ts @@ -7,7 +7,8 @@ */ import util from 'util'; -import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; export class Role { constructor(private log: ToolingLog, private kibanaServer: KbnClient) {} diff --git a/test/common/services/security/role_mappings.ts b/test/common/services/security/role_mappings.ts index c20ff7e327b64..af9204866ad47 100644 --- a/test/common/services/security/role_mappings.ts +++ b/test/common/services/security/role_mappings.ts @@ -7,7 +7,8 @@ */ import util from 'util'; -import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; export class RoleMappings { constructor(private log: ToolingLog, private kbnClient: KbnClient) {} diff --git a/test/common/services/security/user.ts b/test/common/services/security/user.ts index 0d12a0dae2e46..3bd31bb5ed186 100644 --- a/test/common/services/security/user.ts +++ b/test/common/services/security/user.ts @@ -7,7 +7,8 @@ */ import util from 'util'; -import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; export class User { constructor(private log: ToolingLog, private kbnClient: KbnClient) {} diff --git a/test/functional/fixtures/exported_saved_objects/discover.json b/test/functional/fixtures/kbn_archiver/discover.json similarity index 100% rename from test/functional/fixtures/exported_saved_objects/discover.json rename to test/functional/fixtures/kbn_archiver/discover.json diff --git a/x-pack/plugins/case/server/scripts/sub_cases/index.ts b/x-pack/plugins/case/server/scripts/sub_cases/index.ts index 2ea9718d18487..c3bb03242307d 100644 --- a/x-pack/plugins/case/server/scripts/sub_cases/index.ts +++ b/x-pack/plugins/case/server/scripts/sub_cases/index.ts @@ -6,7 +6,8 @@ */ /* eslint-disable no-console */ import yargs from 'yargs'; -import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { CaseResponse, CaseType, diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 3ff719d267f40..96f83f1073fcc 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -7,7 +7,7 @@ import { Client } from '@elastic/elasticsearch'; import seedrandom from 'seedrandom'; -import { KbnClient } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { AxiosResponse } from 'axios'; import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; import { firstNonNullValue } from './models/ecs_safety_helpers'; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts b/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts index 557b485682e15..a8ed7b77a5d9e 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts @@ -7,7 +7,7 @@ import { URL } from 'url'; -import { KbnClient, KbnClientOptions } from '@kbn/dev-utils'; +import { KbnClient, KbnClientOptions } from '@kbn/test'; import fetch, { RequestInit } from 'node-fetch'; export class KbnClientWithApiKeySupport extends KbnClient { diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index a4664e4a51ca2..72f56f13eaddf 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -10,7 +10,8 @@ import yargs from 'yargs'; import fs from 'fs'; import { Client, ClientOptions } from '@elastic/elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; -import { KbnClient, ToolingLog, CA_CERT_PATH } from '@kbn/dev-utils'; +import { ToolingLog, CA_CERT_PATH } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { AxiosResponse } from 'axios'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; import { ANCESTRY_LIMIT, EndpointDocGenerator } from '../../common/endpoint/generate_data'; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/trusted_apps/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/trusted_apps/index.ts index 3b914f987456a..582969f1dcd45 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/trusted_apps/index.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/trusted_apps/index.ts @@ -7,7 +7,8 @@ // @ts-ignore import minimist from 'minimist'; -import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import bluebird from 'bluebird'; import { basename } from 'path'; import { TRUSTED_APPS_CREATE_API, TRUSTED_APPS_LIST_API } from '../../../common/endpoint/constants'; diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 4875e837556fc..1622f211546f5 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { KbnClient } from '@kbn/dev-utils'; +import { KbnClient } from '@kbn/test'; import { ApiResponse, Client } from '@elastic/elasticsearch'; import { SuperTest } from 'supertest'; import supertestAsPromised from 'supertest-as-promised'; From db08fb651f7d9e05e05eb2a9505fdda1a7228a49 Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Feb 2021 12:41:22 -0700 Subject: [PATCH 06/15] pretty print exports and support `--type` in save operations --- packages/kbn-test/src/kbn_archiver_cli.ts | 29 ++++++++- .../kbn_client/kbn_client_import_export.ts | 45 +++++++++---- .../fixtures/kbn_archiver/discover.json | 64 ++++++++++++++++++- 3 files changed, 120 insertions(+), 18 deletions(-) diff --git a/packages/kbn-test/src/kbn_archiver_cli.ts b/packages/kbn-test/src/kbn_archiver_cli.ts index 9a42308602fe8..2d31ecc5e942c 100644 --- a/packages/kbn-test/src/kbn_archiver_cli.ts +++ b/packages/kbn-test/src/kbn_archiver_cli.ts @@ -27,6 +27,22 @@ function getSinglePositionalArg(flags: Flags) { return positional[0]; } +function parseTypesFlag(flags: Flags) { + if (!flags.type) { + return undefined; + } + + if (Array.isArray(flags.type)) { + return flags.type; + } + + if (typeof flags.type === 'string') { + return [flags.type]; + } + + throw createFlagError('--flag must be a string'); +} + export function runKbnArchiverCli() { new RunWithCommands({ description: 'Import/export saved objects from archives, for testing', @@ -96,8 +112,19 @@ export function runKbnArchiverCli() { name: 'save', usage: 'save ', description: 'export saved objects from Kibana to a file', + flags: { + string: ['type'], + help: ` + --type saved object type that should be fetched and stored in the archive, can + be specified multiple times and defaults to 'index-pattern', 'search', + 'visualization', and 'dashboard'. + + `, + }, async run({ kbnClient, flags }) { - await kbnClient.importExport.export(getSinglePositionalArg(flags)); + await kbnClient.importExport.export(getSinglePositionalArg(flags), { + savedObjectTypes: parseTypesFlag(flags), + }); }, }) .command({ diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index c2bf775f60136..832ecbf33e055 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -9,7 +9,6 @@ import { inspect } from 'util'; import Fs from 'fs/promises'; import Path from 'path'; -import { Readable } from 'stream'; import FormData from 'form-data'; import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils'; @@ -17,6 +16,8 @@ import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-util import type { SavedObjectsImportResponse } from 'src/core/server/saved_objects/import/types'; import { KbnClientRequester, uriencode } from './kbn_client_requester'; +const DEFAULT_SAVED_OBJECT_TYPES = ['index-pattern', 'search', 'visualization', 'dashboard']; + export class KbnClientImportExport { constructor( public readonly log: ToolingLog, @@ -24,29 +25,30 @@ export class KbnClientImportExport { public readonly dir?: string ) {} - resolvePath(path: string) { + private resolvePath(path: string) { if (!Path.extname(path)) { path = `${path}.json`; } if (!this.dir && !Path.isAbsolute(path)) { throw new Error( - '[KbnClientImportExport] unable to resolve relative path to import/export without a configured dir' + '[KbnClientImportExport] unable to resolve relative path to import/export without a configured dir, either path absolute path or specify --dir' ); } - return this.dir ? Path.resolve(this.dir, path) : Path.resolve(path); + return this.dir ? Path.resolve(this.dir, path) : path; } async import(name: string, options?: { space?: string }) { const src = this.resolvePath(name); + this.log.debug('resolved import for', name, 'to', src); const objects = (await Fs.readFile(src, 'utf-8')) .split('\n\n') .filter((line) => !!line) .map((line) => JSON.parse(line)); - this.log.debug('importing', objects.length, 'saved objects'); + this.log.info('importing', objects.length, 'saved objects'); const formData = new FormData(); formData.append('file', objects.map((obj) => JSON.stringify(obj)).join('\n'), 'import.ndjson'); @@ -77,7 +79,7 @@ export class KbnClientImportExport { } if (resp.data.success) { - this.log.debug('[KbnClientImportExport] import success'); + this.log.success('[KbnClientImportExport] import success'); } else { throw createFailError( `[KbnClientImportExport] failed to import all saved objects: ${inspect(resp.data)}` @@ -85,15 +87,30 @@ export class KbnClientImportExport { } } - async export(name: string, options?: { space?: string }) { + async export(name: string, options?: { savedObjectTypes?: string[]; space?: string }) { const dest = this.resolvePath(name); - const resp = await this.requester.request({ - method: 'POST', - path: options?.space - ? uriencode`/s/${options.space}/api/saved_objects/_export` - : '/api/saved_objects/_export', - }); + let resp; + try { + resp = await this.requester.request({ + method: 'POST', + path: options?.space + ? uriencode`/s/${options.space}/api/saved_objects/_export` + : '/api/saved_objects/_export', + body: { + type: options?.savedObjectTypes ?? DEFAULT_SAVED_OBJECT_TYPES, + excludeExportDetails: true, + }, + }); + } catch (error) { + if (!isAxiosResponseError(error)) { + throw error; + } + + throw createFailError( + `[KbnClientImportExport] ${error.response.status} resp: ${inspect(error.response.data)}` + ); + } if (typeof resp.data !== 'string' || !resp.data) { throw new Error( @@ -106,6 +123,6 @@ export class KbnClientImportExport { await Fs.writeFile(dest, fileContents, 'utf-8'); - this.log.debug(`[KbnClientImportExport] Exported saved objects to destination:\n ${dest}`); + this.log.success(`[KbnClientImportExport] Exported saved objects to destination:\n ${dest}`); } } diff --git a/test/functional/fixtures/kbn_archiver/discover.json b/test/functional/fixtures/kbn_archiver/discover.json index 578f175642c9d..ec144047a873c 100644 --- a/test/functional/fixtures/kbn_archiver/discover.json +++ b/test/functional/fixtures/kbn_archiver/discover.json @@ -1,4 +1,62 @@ -{"attributes":{"fieldAttrs":"{\"referer\":{\"customLabel\":\"Referer custom\"}}","fields":"[{\"name\":\"@message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@message\"}}},{\"name\":\"@tags\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@tags\"}}},{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"agent\"}}},{\"name\":\"bytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"extension\"}}},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"headings\"}}},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"host\"}}},{\"name\":\"id\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"index\"}}},{\"name\":\"ip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"links\"}}},{\"name\":\"machine.os\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"machine.os\"}}},{\"name\":\"machine.ram\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"esTypes\":[\"double\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"nestedField.child\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"nested\":{\"path\":\"nestedField\"}}},{\"name\":\"phpmemory\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:section\"}}},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:tag\"}}},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:description\"}}},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image\"}}},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:height\"}}},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:width\"}}},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:site_name\"}}},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:title\"}}},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:type\"}}},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:url\"}}},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:card\"}}},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:description\"}}},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:image\"}}},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:site\"}}},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:title\"}}},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.url\"}}},{\"name\":\"request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"request\"}}},{\"name\":\"response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"response\"}}},{\"name\":\"spaces\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"spaces\"}}},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"url\"}}},{\"name\":\"utc_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"xss\"}}}]","timeFieldName":"@timestamp","title":"logstash-*"},"coreMigrationVersion":"8.0.0","id":"logstash-*","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[-9223372036854776000,2],"type":"index-pattern","version":"WzQsMl0="} - -{"attributes":{"columns":["_source"],"description":"A Saved Search Description","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"filter\":[],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"pre712":true,"sort":[["@timestamp","desc"]],"title":"A Saved Search","version":1},"coreMigrationVersion":"8.0.0","id":"ab12e3c0-f231-11e6-9486-733b1ac9221a","migrationVersion":{"search":"7.12.0"},"references":[{"id":"logstash-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[-9223372036854776000,4],"type":"search","version":"WzUsMl0="} +{ + "attributes": { + "fieldAttrs": "{\"referer\":{\"customLabel\":\"Referer custom\"}}", + "fields": "[{\"name\":\"@message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@message\"}}},{\"name\":\"@tags\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@tags\"}}},{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"agent\"}}},{\"name\":\"bytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"extension\"}}},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"headings\"}}},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"host\"}}},{\"name\":\"id\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"index\"}}},{\"name\":\"ip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"links\"}}},{\"name\":\"machine.os\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"machine.os\"}}},{\"name\":\"machine.ram\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"esTypes\":[\"double\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"nestedField.child\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"nested\":{\"path\":\"nestedField\"}}},{\"name\":\"phpmemory\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:section\"}}},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:tag\"}}},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:description\"}}},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image\"}}},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:height\"}}},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:width\"}}},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:site_name\"}}},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:title\"}}},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:type\"}}},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:url\"}}},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:card\"}}},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:description\"}}},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:image\"}}},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:site\"}}},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:title\"}}},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.url\"}}},{\"name\":\"request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"request\"}}},{\"name\":\"response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"response\"}}},{\"name\":\"spaces\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"spaces\"}}},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"url\"}}},{\"name\":\"utc_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"xss\"}}}]", + "timeFieldName": "@timestamp", + "title": "logstash-*" + }, + "coreMigrationVersion": "8.0.0", + "id": "logstash-*", + "migrationVersion": { + "index-pattern": "7.11.0" + }, + "references": [], + "sort": [ + 1613589558181, + 0 + ], + "type": "index-pattern", + "updated_at": "2021-02-17T19:19:18.181Z", + "version": "WzMwLDFd" +} +{ + "attributes": { + "columns": [ + "_source" + ], + "description": "A Saved Search Description", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"highlightAll\":true,\"filter\":[],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "pre712": true, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "A Saved Search", + "version": 1 + }, + "coreMigrationVersion": "8.0.0", + "id": "ab12e3c0-f231-11e6-9486-733b1ac9221a", + "migrationVersion": { + "search": "7.12.0" + }, + "references": [ + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "sort": [ + 1613589558181, + 2 + ], + "type": "search", + "updated_at": "2021-02-17T19:19:18.181Z", + "version": "WzMxLDFd" +} \ No newline at end of file From 7965aa5ca6fb787fd55d75d25f28c92d2af04052 Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Feb 2021 13:25:04 -0700 Subject: [PATCH 07/15] rename methods to load/save --- packages/kbn-test/src/kbn_archiver_cli.ts | 4 ++-- packages/kbn-test/src/kbn_client/kbn_client_import_export.ts | 4 ++-- test/functional/apps/discover/_discover.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/kbn-test/src/kbn_archiver_cli.ts b/packages/kbn-test/src/kbn_archiver_cli.ts index 2d31ecc5e942c..7f94bc7f992ab 100644 --- a/packages/kbn-test/src/kbn_archiver_cli.ts +++ b/packages/kbn-test/src/kbn_archiver_cli.ts @@ -122,7 +122,7 @@ export function runKbnArchiverCli() { `, }, async run({ kbnClient, flags }) { - await kbnClient.importExport.export(getSinglePositionalArg(flags), { + await kbnClient.importExport.save(getSinglePositionalArg(flags), { savedObjectTypes: parseTypesFlag(flags), }); }, @@ -132,7 +132,7 @@ export function runKbnArchiverCli() { usage: 'load ', description: 'import a saved export to Kibana', async run({ kbnClient, flags }) { - await kbnClient.importExport.import(getSinglePositionalArg(flags)); + await kbnClient.importExport.load(getSinglePositionalArg(flags)); }, }) .execute(); diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index 832ecbf33e055..6623c296e451e 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -39,7 +39,7 @@ export class KbnClientImportExport { return this.dir ? Path.resolve(this.dir, path) : path; } - async import(name: string, options?: { space?: string }) { + async load(name: string, options?: { space?: string }) { const src = this.resolvePath(name); this.log.debug('resolved import for', name, 'to', src); @@ -87,7 +87,7 @@ export class KbnClientImportExport { } } - async export(name: string, options?: { savedObjectTypes?: string[]; space?: string }) { + async save(name: string, options?: { savedObjectTypes?: string[]; space?: string }) { const dest = this.resolvePath(name); let resp; diff --git a/test/functional/apps/discover/_discover.ts b/test/functional/apps/discover/_discover.ts index 7d4302968f2df..7127aacac32b6 100644 --- a/test/functional/apps/discover/_discover.ts +++ b/test/functional/apps/discover/_discover.ts @@ -30,7 +30,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { log.debug('load kibana index with default index pattern'); await esArchiver.load('empty_kibana'); - await kibanaServer.importExport.import('discover'); + await kibanaServer.importExport.load('discover'); // and load a set of makelogs data await esArchiver.loadIfNeeded('logstash_functional'); From 68921747f65eb7bbc5c896d2bab2feeb76dc70b7 Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Feb 2021 13:44:24 -0700 Subject: [PATCH 08/15] avoid importing types from src --- packages/kbn-test/src/kbn_client/kbn_client_import_export.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index 6623c296e451e..84dc356ea4039 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -13,7 +13,6 @@ import Path from 'path'; import FormData from 'form-data'; import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils'; -import type { SavedObjectsImportResponse } from 'src/core/server/saved_objects/import/types'; import { KbnClientRequester, uriencode } from './kbn_client_requester'; const DEFAULT_SAVED_OBJECT_TYPES = ['index-pattern', 'search', 'visualization', 'dashboard']; @@ -57,7 +56,7 @@ export class KbnClientImportExport { let resp; try { - resp = await this.requester.request({ + resp = await this.requester.request<{ success: boolean; [key: string]: unknown }>({ method: 'POST', path: options?.space ? uriencode`/s/${options.space}/api/saved_objects/_import` From 9b2f4374b1a0bd1d54964d50f3777fb203ed4295 Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Feb 2021 14:29:30 -0700 Subject: [PATCH 09/15] add unload method --- packages/kbn-test/src/kbn_archiver_cli.ts | 8 ++ .../kbn_client/kbn_client_import_export.ts | 81 ++++++++++++++++--- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/packages/kbn-test/src/kbn_archiver_cli.ts b/packages/kbn-test/src/kbn_archiver_cli.ts index 7f94bc7f992ab..419a544a2cc16 100644 --- a/packages/kbn-test/src/kbn_archiver_cli.ts +++ b/packages/kbn-test/src/kbn_archiver_cli.ts @@ -135,5 +135,13 @@ export function runKbnArchiverCli() { await kbnClient.importExport.load(getSinglePositionalArg(flags)); }, }) + .command({ + name: 'unload', + usage: 'unload ', + description: 'delete the saved objects saved in the archive from the Kibana index', + async run({ kbnClient, flags }) { + await kbnClient.importExport.unload(getSinglePositionalArg(flags)); + }, + }) .execute(); } diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index 84dc356ea4039..f0e12d197f7f3 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -10,6 +10,9 @@ import { inspect } from 'util'; import Fs from 'fs/promises'; import Path from 'path'; +import * as Rx from 'rxjs'; +import { mergeMap } from 'rxjs/operators'; +import { lastValueFrom } from '@kbn/std'; import FormData from 'form-data'; import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils'; @@ -17,6 +20,23 @@ import { KbnClientRequester, uriencode } from './kbn_client_requester'; const DEFAULT_SAVED_OBJECT_TYPES = ['index-pattern', 'search', 'visualization', 'dashboard']; +interface SavedObject { + id: string; + type: string; + [key: string]: unknown; +} + +async function parseArchive(path: string): Promise { + return (await Fs.readFile(path, 'utf-8')) + .split('\n\n') + .filter((line) => !!line) + .map((line) => JSON.parse(line)); +} + +async function concurrently(maxConcurrency: number, arr: T[], fn: (item: T) => Promise) { + await lastValueFrom(Rx.from(arr).pipe(mergeMap(async (item) => await fn(item), maxConcurrency))); +} + export class KbnClientImportExport { constructor( public readonly log: ToolingLog, @@ -42,11 +62,7 @@ export class KbnClientImportExport { const src = this.resolvePath(name); this.log.debug('resolved import for', name, 'to', src); - const objects = (await Fs.readFile(src, 'utf-8')) - .split('\n\n') - .filter((line) => !!line) - .map((line) => JSON.parse(line)); - + const objects = await parseArchive(src); this.log.info('importing', objects.length, 'saved objects'); const formData = new FormData(); @@ -86,8 +102,51 @@ export class KbnClientImportExport { } } + async unload(name: string, options?: { space?: string }) { + const src = this.resolvePath(name); + this.log.debug('unloading docs from archive at', src); + + const objects = await parseArchive(src); + this.log.info('deleting', objects.length, 'objects'); + + let deleted = 0; + let missing = 0; + + await concurrently(20, objects, async (obj) => { + try { + await this.requester.request({ + method: 'DELETE', + path: options?.space + ? uriencode`/s/${options.space}/api/saved_objects/${obj.type}/${obj.id}` + : uriencode`/api/saved_objects/${obj.type}/${obj.id}`, + }); + deleted++; + } catch (error) { + if (isAxiosResponseError(error)) { + if (error.response.status === 404) { + missing++; + return; + } + + throw createFailError( + `[KbnClientImportExport] ${error.response.status} resp: ${inspect(error.response.data)}` + ); + } + + throw error; + } + }); + + if (missing) { + this.log.info(missing, 'saved objects were already deleted'); + } + + this.log.success(deleted, 'saved objects deleted'); + } + async save(name: string, options?: { savedObjectTypes?: string[]; space?: string }) { const dest = this.resolvePath(name); + this.log.debug('saving export to', dest); let resp; try { @@ -111,17 +170,21 @@ export class KbnClientImportExport { ); } - if (typeof resp.data !== 'string' || !resp.data) { - throw new Error( + if (typeof resp.data !== 'string') { + throw createFailError( `[KbnClientImportExport] unexpected response from export API: ${inspect(resp.data)}` ); } - const objects = resp.data.split('\n').map((line) => JSON.parse(line)); + const objects = resp.data + .split('\n') + .filter((l) => !!l) + .map((line) => JSON.parse(line)); + const fileContents = objects.map((obj) => JSON.stringify(obj, null, 2)).join('\n\n'); await Fs.writeFile(dest, fileContents, 'utf-8'); - this.log.success(`[KbnClientImportExport] Exported saved objects to destination:\n ${dest}`); + this.log.success('[KbnClientImportExport] Exported', objects.length, 'saved objects to', dest); } } From a7396106548c58856f0af4ca62c375809e8d0871 Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Feb 2021 14:34:19 -0700 Subject: [PATCH 10/15] fix kbnArchiver.directory config getter --- test/common/services/kibana_server/kibana_server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/services/kibana_server/kibana_server.ts b/test/common/services/kibana_server/kibana_server.ts index 5b1ea1429873a..f366a864db980 100644 --- a/test/common/services/kibana_server/kibana_server.ts +++ b/test/common/services/kibana_server/kibana_server.ts @@ -22,7 +22,7 @@ export function KibanaServerProvider({ getService }: FtrProviderContext) { url, certificateAuthorities: config.get('servers.kibana.certificateAuthorities'), uiSettingDefaults: defaults, - importExportDir: config.get('savedObjects.directory'), + importExportDir: config.get('kbnArchiver.directory'), }); if (defaults) { From 31fbc10cf97f74f95d0b607ea229cf553cfe3c55 Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Feb 2021 18:05:01 -0700 Subject: [PATCH 11/15] add clean() method for deleting objects by type --- packages/kbn-test/src/kbn_archiver_cli.ts | 59 ++++-- .../kbn_client/kbn_client_import_export.ts | 195 +++++++++++------- 2 files changed, 154 insertions(+), 100 deletions(-) diff --git a/packages/kbn-test/src/kbn_archiver_cli.ts b/packages/kbn-test/src/kbn_archiver_cli.ts index 419a544a2cc16..cd843919cb249 100644 --- a/packages/kbn-test/src/kbn_archiver_cli.ts +++ b/packages/kbn-test/src/kbn_archiver_cli.ts @@ -28,19 +28,15 @@ function getSinglePositionalArg(flags: Flags) { } function parseTypesFlag(flags: Flags) { - if (!flags.type) { - return undefined; + if (!flags.type || (typeof flags.type !== 'string' && !Array.isArray(flags.type))) { + throw createFlagError('--type is a required flag'); } - if (Array.isArray(flags.type)) { - return flags.type; - } - - if (typeof flags.type === 'string') { - return [flags.type]; - } - - throw createFlagError('--flag must be a string'); + const types = typeof flags.type === 'string' ? [flags.type] : flags.type; + return types.reduce( + (acc: string[], type) => [...acc, ...type.split(',').map((t) => t.trim())], + [] + ); } export function runKbnArchiverCli() { @@ -49,6 +45,7 @@ export function runKbnArchiverCli() { globalFlags: { string: ['config'], help: ` + --space space id to operate on, defaults to the default space --config optional path to an FTR config file that will be parsed and used for defaults --kibana-url set the url that kibana can be reached at, uses the "servers.kibana" setting from --config by default --dir directory that contains exports to be imported, or where exports will be saved, uses the "kbnArchiver.directory" @@ -99,7 +96,13 @@ export function runKbnArchiverCli() { ); } + const space = flags.space; + if (!(space === undefined || typeof space === 'string')) { + throw createFlagError('--space must be a string'); + } + return { + space, kbnClient: new KbnClient({ log, url: kibanaUrl, @@ -116,14 +119,13 @@ export function runKbnArchiverCli() { string: ['type'], help: ` --type saved object type that should be fetched and stored in the archive, can - be specified multiple times and defaults to 'index-pattern', 'search', - 'visualization', and 'dashboard'. - + be specified multiple times or be a comma-separated list. `, }, - async run({ kbnClient, flags }) { + async run({ kbnClient, flags, space }) { await kbnClient.importExport.save(getSinglePositionalArg(flags), { - savedObjectTypes: parseTypesFlag(flags), + types: parseTypesFlag(flags), + space, }); }, }) @@ -131,16 +133,33 @@ export function runKbnArchiverCli() { name: 'load', usage: 'load ', description: 'import a saved export to Kibana', - async run({ kbnClient, flags }) { - await kbnClient.importExport.load(getSinglePositionalArg(flags)); + async run({ kbnClient, flags, space }) { + await kbnClient.importExport.load(getSinglePositionalArg(flags), { space }); }, }) .command({ name: 'unload', usage: 'unload ', description: 'delete the saved objects saved in the archive from the Kibana index', - async run({ kbnClient, flags }) { - await kbnClient.importExport.unload(getSinglePositionalArg(flags)); + async run({ kbnClient, flags, space }) { + await kbnClient.importExport.unload(getSinglePositionalArg(flags), { space }); + }, + }) + .command({ + name: 'clean', + description: 'clean all saved objects of specific types from the Kibana index', + flags: { + string: ['type'], + help: ` + --type saved object type that should be cleaned from the index, can + be specified multiple times or be a comma-separated list. + `, + }, + async run({ kbnClient, space, flags }) { + await kbnClient.importExport.clean({ + types: parseTypesFlag(flags), + space, + }); }, }) .execute(); diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index f0e12d197f7f3..d51e2a6c7bfa5 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -16,9 +16,19 @@ import { lastValueFrom } from '@kbn/std'; import FormData from 'form-data'; import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils'; -import { KbnClientRequester, uriencode } from './kbn_client_requester'; +import { KbnClientRequester, uriencode, ReqOptions } from './kbn_client_requester'; -const DEFAULT_SAVED_OBJECT_TYPES = ['index-pattern', 'search', 'visualization', 'dashboard']; +interface ImportApiResponse { + success: boolean; + [key: string]: unknown; +} + +interface FindApiResponse { + saved_objects: SavedObject[]; + total: number; + per_page: number; + page: number; +} interface SavedObject { id: string; @@ -51,7 +61,7 @@ export class KbnClientImportExport { if (!this.dir && !Path.isAbsolute(path)) { throw new Error( - '[KbnClientImportExport] unable to resolve relative path to import/export without a configured dir, either path absolute path or specify --dir' + 'unable to resolve relative path to import/export without a configured dir, either path absolute path or specify --dir' ); } @@ -63,43 +73,55 @@ export class KbnClientImportExport { this.log.debug('resolved import for', name, 'to', src); const objects = await parseArchive(src); - this.log.info('importing', objects.length, 'saved objects'); + this.log.info('importing', objects.length, 'saved objects', { space: options?.space }); const formData = new FormData(); formData.append('file', objects.map((obj) => JSON.stringify(obj)).join('\n'), 'import.ndjson'); // TODO: should we clear out the existing saved objects? + const resp = await this.req(options?.space, { + method: 'POST', + path: '/api/saved_objects/_import', + query: { + overwrite: true, + }, + body: formData, + headers: formData.getHeaders(), + }); - let resp; - try { - resp = await this.requester.request<{ success: boolean; [key: string]: unknown }>({ - method: 'POST', - path: options?.space - ? uriencode`/s/${options.space}/api/saved_objects/_import` - : '/api/saved_objects/_import', + if (resp.data.success) { + this.log.success('import success'); + } else { + throw createFailError(`failed to import all saved objects: ${inspect(resp.data)}`); + } + } + + async clean(options: { types: string[]; space?: string }) { + this.log.debug('cleaning all saved objects', { space: options?.space }); + + let deleted = 0; + + while (true) { + const resp = await this.req(options.space, { + method: 'GET', + path: '/api/saved_objects/_find', query: { - overwrite: true, + per_page: 1000, + type: options.types, + fields: 'none', }, - body: formData, - headers: formData.getHeaders(), }); - } catch (error) { - if (!isAxiosResponseError(error)) { - throw error; - } - throw createFailError( - `[KbnClientImportExport] ${error.response.status} resp: ${inspect(error.response.data)}` - ); - } + this.log.info('deleting batch of', resp.data.saved_objects.length, 'objects'); + const deletion = await this.deleteObjects(options.space, resp.data.saved_objects); + deleted += deletion.deleted; - if (resp.data.success) { - this.log.success('[KbnClientImportExport] import success'); - } else { - throw createFailError( - `[KbnClientImportExport] failed to import all saved objects: ${inspect(resp.data)}` - ); + if (resp.data.total < resp.data.per_page) { + break; + } } + + this.log.success('deleted', deleted, 'objects'); } async unload(name: string, options?: { space?: string }) { @@ -107,35 +129,9 @@ export class KbnClientImportExport { this.log.debug('unloading docs from archive at', src); const objects = await parseArchive(src); - this.log.info('deleting', objects.length, 'objects'); - - let deleted = 0; - let missing = 0; - - await concurrently(20, objects, async (obj) => { - try { - await this.requester.request({ - method: 'DELETE', - path: options?.space - ? uriencode`/s/${options.space}/api/saved_objects/${obj.type}/${obj.id}` - : uriencode`/api/saved_objects/${obj.type}/${obj.id}`, - }); - deleted++; - } catch (error) { - if (isAxiosResponseError(error)) { - if (error.response.status === 404) { - missing++; - return; - } - - throw createFailError( - `[KbnClientImportExport] ${error.response.status} resp: ${inspect(error.response.data)}` - ); - } + this.log.info('deleting', objects.length, 'objects', { space: options?.space }); - throw error; - } - }); + const { deleted, missing } = await this.deleteObjects(options?.space, objects); if (missing) { this.log.info(missing, 'saved objects were already deleted'); @@ -144,21 +140,45 @@ export class KbnClientImportExport { this.log.success(deleted, 'saved objects deleted'); } - async save(name: string, options?: { savedObjectTypes?: string[]; space?: string }) { + async save(name: string, options: { types: string[]; space?: string }) { const dest = this.resolvePath(name); this.log.debug('saving export to', dest); - let resp; + const resp = await this.req(options.space, { + method: 'POST', + path: '/api/saved_objects/_export', + body: { + type: options.types, + excludeExportDetails: true, + includeReferencesDeep: true, + }, + }); + + if (typeof resp.data !== 'string') { + throw createFailError(`unexpected response from export API: ${inspect(resp.data)}`); + } + + const objects = resp.data + .split('\n') + .filter((l) => !!l) + .map((line) => JSON.parse(line)); + + const fileContents = objects.map((obj) => JSON.stringify(obj, null, 2)).join('\n\n'); + + await Fs.writeFile(dest, fileContents, 'utf-8'); + + this.log.success('Exported', objects.length, 'saved objects to', dest); + } + + private async req(space: string | undefined, options: ReqOptions) { + if (!options.path.startsWith('/')) { + throw new Error('options.path must start with a /'); + } + try { - resp = await this.requester.request({ - method: 'POST', - path: options?.space - ? uriencode`/s/${options.space}/api/saved_objects/_export` - : '/api/saved_objects/_export', - body: { - type: options?.savedObjectTypes ?? DEFAULT_SAVED_OBJECT_TYPES, - excludeExportDetails: true, - }, + return await this.requester.request({ + ...options, + path: space ? uriencode`/s/${space}` + options.path : options.path, }); } catch (error) { if (!isAxiosResponseError(error)) { @@ -166,25 +186,40 @@ export class KbnClientImportExport { } throw createFailError( - `[KbnClientImportExport] ${error.response.status} resp: ${inspect(error.response.data)}` + `${error.response.status} resp: ${inspect(error.response.data)}\nreq: ${inspect( + error.config + )}` ); } + } - if (typeof resp.data !== 'string') { - throw createFailError( - `[KbnClientImportExport] unexpected response from export API: ${inspect(resp.data)}` - ); - } + private async deleteObjects(space: string | undefined, objects: SavedObject[]) { + let deleted = 0; + let missing = 0; - const objects = resp.data - .split('\n') - .filter((l) => !!l) - .map((line) => JSON.parse(line)); + await concurrently(20, objects, async (obj) => { + try { + await this.requester.request({ + method: 'DELETE', + path: space + ? uriencode`/s/${space}/api/saved_objects/${obj.type}/${obj.id}` + : uriencode`/api/saved_objects/${obj.type}/${obj.id}`, + }); + deleted++; + } catch (error) { + if (isAxiosResponseError(error)) { + if (error.response.status === 404) { + missing++; + return; + } - const fileContents = objects.map((obj) => JSON.stringify(obj, null, 2)).join('\n\n'); + throw createFailError(`${error.response.status} resp: ${inspect(error.response.data)}`); + } - await Fs.writeFile(dest, fileContents, 'utf-8'); + throw error; + } + }); - this.log.success('[KbnClientImportExport] Exported', objects.length, 'saved objects to', dest); + return { deleted, missing }; } } From 9dd92c8e0844d7fe7bdade8901e3394ea94276d1 Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Feb 2021 18:32:43 -0700 Subject: [PATCH 12/15] move clean and bulkDelete to savedObjects client --- packages/kbn-test/src/kbn_archiver_cli.ts | 17 --- .../kbn_client/kbn_client_import_export.ts | 79 ++------------ .../src/kbn_client/kbn_client_requester.ts | 2 + .../kbn_client/kbn_client_saved_objects.ts | 102 +++++++++++++++++- test/functional/apps/discover/_discover.ts | 2 +- 5 files changed, 110 insertions(+), 92 deletions(-) diff --git a/packages/kbn-test/src/kbn_archiver_cli.ts b/packages/kbn-test/src/kbn_archiver_cli.ts index cd843919cb249..6431e7d8f1976 100644 --- a/packages/kbn-test/src/kbn_archiver_cli.ts +++ b/packages/kbn-test/src/kbn_archiver_cli.ts @@ -145,22 +145,5 @@ export function runKbnArchiverCli() { await kbnClient.importExport.unload(getSinglePositionalArg(flags), { space }); }, }) - .command({ - name: 'clean', - description: 'clean all saved objects of specific types from the Kibana index', - flags: { - string: ['type'], - help: ` - --type saved object type that should be cleaned from the index, can - be specified multiple times or be a comma-separated list. - `, - }, - async run({ kbnClient, space, flags }) { - await kbnClient.importExport.clean({ - types: parseTypesFlag(flags), - space, - }); - }, - }) .execute(); } diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index d51e2a6c7bfa5..6374b8aac3688 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -10,26 +10,17 @@ import { inspect } from 'util'; import Fs from 'fs/promises'; import Path from 'path'; -import * as Rx from 'rxjs'; -import { mergeMap } from 'rxjs/operators'; -import { lastValueFrom } from '@kbn/std'; import FormData from 'form-data'; import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils'; import { KbnClientRequester, uriencode, ReqOptions } from './kbn_client_requester'; +import { KbnClientSavedObjects } from './kbn_client_saved_objects'; interface ImportApiResponse { success: boolean; [key: string]: unknown; } -interface FindApiResponse { - saved_objects: SavedObject[]; - total: number; - per_page: number; - page: number; -} - interface SavedObject { id: string; type: string; @@ -43,14 +34,11 @@ async function parseArchive(path: string): Promise { .map((line) => JSON.parse(line)); } -async function concurrently(maxConcurrency: number, arr: T[], fn: (item: T) => Promise) { - await lastValueFrom(Rx.from(arr).pipe(mergeMap(async (item) => await fn(item), maxConcurrency))); -} - export class KbnClientImportExport { constructor( public readonly log: ToolingLog, public readonly requester: KbnClientRequester, + public readonly savedObjects: KbnClientSavedObjects, public readonly dir?: string ) {} @@ -96,34 +84,6 @@ export class KbnClientImportExport { } } - async clean(options: { types: string[]; space?: string }) { - this.log.debug('cleaning all saved objects', { space: options?.space }); - - let deleted = 0; - - while (true) { - const resp = await this.req(options.space, { - method: 'GET', - path: '/api/saved_objects/_find', - query: { - per_page: 1000, - type: options.types, - fields: 'none', - }, - }); - - this.log.info('deleting batch of', resp.data.saved_objects.length, 'objects'); - const deletion = await this.deleteObjects(options.space, resp.data.saved_objects); - deleted += deletion.deleted; - - if (resp.data.total < resp.data.per_page) { - break; - } - } - - this.log.success('deleted', deleted, 'objects'); - } - async unload(name: string, options?: { space?: string }) { const src = this.resolvePath(name); this.log.debug('unloading docs from archive at', src); @@ -131,7 +91,10 @@ export class KbnClientImportExport { const objects = await parseArchive(src); this.log.info('deleting', objects.length, 'objects', { space: options?.space }); - const { deleted, missing } = await this.deleteObjects(options?.space, objects); + const { deleted, missing } = await this.savedObjects.bulkDelete({ + space: options?.space, + objects, + }); if (missing) { this.log.info(missing, 'saved objects were already deleted'); @@ -192,34 +155,4 @@ export class KbnClientImportExport { ); } } - - private async deleteObjects(space: string | undefined, objects: SavedObject[]) { - let deleted = 0; - let missing = 0; - - await concurrently(20, objects, async (obj) => { - try { - await this.requester.request({ - method: 'DELETE', - path: space - ? uriencode`/s/${space}/api/saved_objects/${obj.type}/${obj.id}` - : uriencode`/api/saved_objects/${obj.type}/${obj.id}`, - }); - deleted++; - } catch (error) { - if (isAxiosResponseError(error)) { - if (error.response.status === 404) { - missing++; - return; - } - - throw createFailError(`${error.response.status} resp: ${inspect(error.response.data)}`); - } - - throw error; - } - }); - - return { deleted, missing }; - } } diff --git a/packages/kbn-test/src/kbn_client/kbn_client_requester.ts b/packages/kbn-test/src/kbn_client/kbn_client_requester.ts index 8bdfe7da35ddc..2e1575aee1897 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_requester.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_requester.ts @@ -8,6 +8,7 @@ import Url from 'url'; import Https from 'https'; +import Qs from 'querystring'; import Axios, { AxiosResponse } from 'axios'; import { ToolingLog, isAxiosRequestError, isAxiosResponseError } from '@kbn/dev-utils'; @@ -106,6 +107,7 @@ export class KbnClientRequester { 'kbn-xsrf': 'kbn-client', }, httpsAgent: this.httpsAgent, + paramsSerializer: (params) => Qs.stringify(params), }); return response; diff --git a/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts b/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts index b175e979631ee..904ccc385bd7d 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts @@ -6,7 +6,12 @@ * Side Public License, v 1. */ -import { ToolingLog } from '@kbn/dev-utils'; +import { inspect } from 'util'; + +import * as Rx from 'rxjs'; +import { mergeMap } from 'rxjs/operators'; +import { lastValueFrom } from '@kbn/std'; +import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils'; import { KbnClientRequester, uriencode } from './kbn_client_requester'; @@ -51,6 +56,38 @@ interface MigrateResponse { result: Array<{ status: string }>; } +interface FindApiResponse { + saved_objects: Array<{ + type: string; + id: string; + [key: string]: unknown; + }>; + total: number; + per_page: number; + page: number; +} + +interface CleanOptions { + space?: string; + types: string[]; +} + +interface DeleteObjectsOptions { + space?: string; + objects: Array<{ + type: string; + id: string; + }>; +} + +async function concurrently(maxConcurrency: number, arr: T[], fn: (item: T) => Promise) { + if (arr.length) { + await lastValueFrom( + Rx.from(arr).pipe(mergeMap(async (item) => await fn(item), maxConcurrency)) + ); + } +} + export class KbnClientSavedObjects { constructor(private readonly log: ToolingLog, private readonly requester: KbnClientRequester) {} @@ -143,4 +180,67 @@ export class KbnClientSavedObjects { return data; } + + public async clean(options: CleanOptions) { + this.log.debug('Cleaning all saved objects', { space: options.space }); + + let deleted = 0; + + while (true) { + const resp = await this.requester.request({ + method: 'GET', + path: options.space + ? uriencode`/s/${options.space}/api/saved_objects/_find` + : '/api/saved_objects/_find', + query: { + per_page: 1000, + type: options.types, + fields: 'none', + }, + }); + + this.log.info('deleting batch of', resp.data.saved_objects.length, 'objects'); + const deletion = await this.bulkDelete({ + space: options.space, + objects: resp.data.saved_objects, + }); + deleted += deletion.deleted; + + if (resp.data.total <= resp.data.per_page) { + break; + } + } + + this.log.success('deleted', deleted, 'objects'); + } + + public async bulkDelete(options: DeleteObjectsOptions) { + let deleted = 0; + let missing = 0; + + await concurrently(20, options.objects, async (obj) => { + try { + await this.requester.request({ + method: 'DELETE', + path: options.space + ? uriencode`/s/${options.space}/api/saved_objects/${obj.type}/${obj.id}` + : uriencode`/api/saved_objects/${obj.type}/${obj.id}`, + }); + deleted++; + } catch (error) { + if (isAxiosResponseError(error)) { + if (error.response.status === 404) { + missing++; + return; + } + + throw createFailError(`${error.response.status} resp: ${inspect(error.response.data)}`); + } + + throw error; + } + }); + + return { deleted, missing }; + } } diff --git a/test/functional/apps/discover/_discover.ts b/test/functional/apps/discover/_discover.ts index 7127aacac32b6..aeb02e5c30eb8 100644 --- a/test/functional/apps/discover/_discover.ts +++ b/test/functional/apps/discover/_discover.ts @@ -29,7 +29,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async function () { log.debug('load kibana index with default index pattern'); - await esArchiver.load('empty_kibana'); + await kibanaServer.savedObjects.clean({ types: ['search'] }); await kibanaServer.importExport.load('discover'); // and load a set of makelogs data From 346d6fa5fdb68ba898b3d0b73309f672d1238cbc Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 22 Feb 2021 10:18:41 -0700 Subject: [PATCH 13/15] pass KbnClientSavedObjects to KbnClientImportExport --- packages/kbn-test/src/kbn_client/kbn_client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-test/src/kbn_client/kbn_client.ts b/packages/kbn-test/src/kbn_client/kbn_client.ts index 08092f32bebb1..3fa74412c1a8b 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client.ts @@ -63,6 +63,7 @@ export class KbnClient { this.importExport = new KbnClientImportExport( this.log, this.requester, + this.savedObjects, options.importExportDir ); } From 99eba13025504fd9c74e3de786de4c1ac764b5b6 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 22 Feb 2021 13:19:54 -0700 Subject: [PATCH 14/15] strip sort field from exports --- .../kbn-test/src/kbn_client/kbn_client_import_export.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index 6374b8aac3688..bb5b99fdc4439 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -126,7 +126,12 @@ export class KbnClientImportExport { .filter((l) => !!l) .map((line) => JSON.parse(line)); - const fileContents = objects.map((obj) => JSON.stringify(obj, null, 2)).join('\n\n'); + const fileContents = objects + .map((obj) => { + const { sort: _, ...nonSortFields } = obj; + return JSON.stringify(nonSortFields, null, 2); + }) + .join('\n\n'); await Fs.writeFile(dest, fileContents, 'utf-8'); From cd6a509ae6ebbea60179f6742f6651830ecf9643 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 22 Feb 2021 13:20:36 -0700 Subject: [PATCH 15/15] rebuild discover export from esArchive --- .../fixtures/kbn_archiver/discover.json | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/test/functional/fixtures/kbn_archiver/discover.json b/test/functional/fixtures/kbn_archiver/discover.json index ec144047a873c..e861f875a2d9e 100644 --- a/test/functional/fixtures/kbn_archiver/discover.json +++ b/test/functional/fixtures/kbn_archiver/discover.json @@ -11,13 +11,8 @@ "index-pattern": "7.11.0" }, "references": [], - "sort": [ - 1613589558181, - 0 - ], "type": "index-pattern", - "updated_at": "2021-02-17T19:19:18.181Z", - "version": "WzMwLDFd" + "version": "WzQsMl0=" } { @@ -30,7 +25,6 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"highlightAll\":true,\"filter\":[],\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, - "pre712": true, "sort": [ [ "@timestamp", @@ -43,7 +37,7 @@ "coreMigrationVersion": "8.0.0", "id": "ab12e3c0-f231-11e6-9486-733b1ac9221a", "migrationVersion": { - "search": "7.12.0" + "search": "7.9.3" }, "references": [ { @@ -52,11 +46,6 @@ "type": "index-pattern" } ], - "sort": [ - 1613589558181, - 2 - ], "type": "search", - "updated_at": "2021-02-17T19:19:18.181Z", - "version": "WzMxLDFd" + "version": "WzUsMl0=" } \ No newline at end of file