[QA] Create Saved Objects Kibana Svc

packages/kbn-test/src/kbn_archiver_cli.ts

@@ -28,19 +28,15 @@ function getSinglePositionalArg(flags: Flags) {
 }
 
 function parseTypesFlag(flags: Flags) {
-  if (!flags.type) {
-    return undefined;
+  if (!flags.type || (typeof flags.type !== 'string' && !Array.isArray(flags.type))) {
+    throw createFlagError('--type is a required flag');
   }
 
-  if (Array.isArray(flags.type)) {
-    return flags.type;
-  }
-
-  if (typeof flags.type === 'string') {
-    return [flags.type];
-  }
-
-  throw createFlagError('--flag must be a string');
+  const types = typeof flags.type === 'string' ? [flags.type] : flags.type;
+  return types.reduce(
+    (acc: string[], type) => [...acc, ...type.split(',').map((t) => t.trim())],
+    []
+  );
 }
 
 export function runKbnArchiverCli() {
@@ -49,6 +45,7 @@ export function runKbnArchiverCli() {
     globalFlags: {
       string: ['config'],
       help: `
+        --space            space id to operate on, defaults to the default space
         --config           optional path to an FTR config file that will be parsed and used for defaults
         --kibana-url       set the url that kibana can be reached at, uses the "servers.kibana" setting from --config by default
         --dir              directory that contains exports to be imported, or where exports will be saved, uses the "kbnArchiver.directory"
@@ -99,7 +96,13 @@ export function runKbnArchiverCli() {
         );
       }
 
+      const space = flags.space;
+      if (!(space === undefined || typeof space === 'string')) {
+        throw createFlagError('--space must be a string');
+      }
+
       return {
+        space,
         kbnClient: new KbnClient({
           log,
           url: kibanaUrl,
@@ -116,31 +119,47 @@ export function runKbnArchiverCli() {
         string: ['type'],
         help: `
           --type             saved object type that should be fetched and stored in the archive, can
-                               be specified multiple times and defaults to 'index-pattern', 'search',
-                               'visualization', and 'dashboard'.
-
+                               be specified multiple times or be a comma-separated list.
         `,
       },
-      async run({ kbnClient, flags }) {
+      async run({ kbnClient, flags, space }) {
         await kbnClient.importExport.save(getSinglePositionalArg(flags), {
-          savedObjectTypes: parseTypesFlag(flags),
+          types: parseTypesFlag(flags),
+          space,
         });
       },
     })
     .command({
       name: 'load',
       usage: 'load <name>',
       description: 'import a saved export to Kibana',
-      async run({ kbnClient, flags }) {
-        await kbnClient.importExport.load(getSinglePositionalArg(flags));
+      async run({ kbnClient, flags, space }) {
+        await kbnClient.importExport.load(getSinglePositionalArg(flags), { space });
       },
     })
     .command({
       name: 'unload',
       usage: 'unload <name>',
       description: 'delete the saved objects saved in the archive from the Kibana index',
-      async run({ kbnClient, flags }) {
-        await kbnClient.importExport.unload(getSinglePositionalArg(flags));
+      async run({ kbnClient, flags, space }) {
+        await kbnClient.importExport.unload(getSinglePositionalArg(flags), { space });
+      },
+    })
+    .command({
+      name: 'clean',
+      description: 'clean all saved objects of specific types from the Kibana index',
+      flags: {
+        string: ['type'],
+        help: `
+          --type             saved object type that should be cleaned from the index, can
+                               be specified multiple times or be a comma-separated list.
+        `,
+      },
+      async run({ kbnClient, space, flags }) {
+        await kbnClient.importExport.clean({
+          types: parseTypesFlag(flags),
+          space,
+        });
       },
     })
     .execute();

packages/kbn-test/src/kbn_client/kbn_client_import_export.ts

@@ -16,9 +16,19 @@ import { lastValueFrom } from '@kbn/std';
 import FormData from 'form-data';
 import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils';
 
-import { KbnClientRequester, uriencode } from './kbn_client_requester';
+import { KbnClientRequester, uriencode, ReqOptions } from './kbn_client_requester';
 
-const DEFAULT_SAVED_OBJECT_TYPES = ['index-pattern', 'search', 'visualization', 'dashboard'];
+interface ImportApiResponse {
+  success: boolean;
+  [key: string]: unknown;
+}
+
+interface FindApiResponse {
+  saved_objects: SavedObject[];
+  total: number;
+  per_page: number;
+  page: number;
+}
 
 interface SavedObject {
   id: string;
@@ -51,7 +61,7 @@ export class KbnClientImportExport {
 
     if (!this.dir && !Path.isAbsolute(path)) {
       throw new Error(
-        '[KbnClientImportExport] unable to resolve relative path to import/export without a configured dir, either path absolute path or specify --dir'
+        'unable to resolve relative path to import/export without a configured dir, either path absolute path or specify --dir'
       );
     }
 
@@ -63,79 +73,65 @@ export class KbnClientImportExport {
     this.log.debug('resolved import for', name, 'to', src);
 
     const objects = await parseArchive(src);
-    this.log.info('importing', objects.length, 'saved objects');
+    this.log.info('importing', objects.length, 'saved objects', { space: options?.space });
 
     const formData = new FormData();
     formData.append('file', objects.map((obj) => JSON.stringify(obj)).join('\n'), 'import.ndjson');
 
     // TODO: should we clear out the existing saved objects?
+    const resp = await this.req<ImportApiResponse>(options?.space, {
+      method: 'POST',
+      path: '/api/saved_objects/_import',
+      query: {
+        overwrite: true,
+      },
+      body: formData,
+      headers: formData.getHeaders(),
+    });
 
-    let resp;
-    try {
-      resp = await this.requester.request<{ success: boolean; [key: string]: unknown }>({
-        method: 'POST',
-        path: options?.space
-          ? uriencode`/s/${options.space}/api/saved_objects/_import`
-          : '/api/saved_objects/_import',
+    if (resp.data.success) {
+      this.log.success('import success');
+    } else {
+      throw createFailError(`failed to import all saved objects: ${inspect(resp.data)}`);
+    }
+  }
+
+  async clean(options: { types: string[]; space?: string }) {
+    this.log.debug('cleaning all saved objects', { space: options?.space });
+
+    let deleted = 0;
+
+    while (true) {
+      const resp = await this.req<FindApiResponse>(options.space, {
+        method: 'GET',
+        path: '/api/saved_objects/_find',
         query: {
-          overwrite: true,
+          per_page: 1000,
+          type: options.types,
+          fields: 'none',
         },
-        body: formData,
-        headers: formData.getHeaders(),
       });
-    } catch (error) {
-      if (!isAxiosResponseError(error)) {
-        throw error;
-      }
 
-      throw createFailError(
-        `[KbnClientImportExport] ${error.response.status} resp: ${inspect(error.response.data)}`
-      );
-    }
+      this.log.info('deleting batch of', resp.data.saved_objects.length, 'objects');
+      const deletion = await this.deleteObjects(options.space, resp.data.saved_objects);
+      deleted += deletion.deleted;
 
-    if (resp.data.success) {
-      this.log.success('[KbnClientImportExport] import success');
-    } else {
-      throw createFailError(
-        `[KbnClientImportExport] failed to import all saved objects: ${inspect(resp.data)}`
-      );
+      if (resp.data.total < resp.data.per_page) {
+        break;
+      }
     }
+
+    this.log.success('deleted', deleted, 'objects');
   }
 
   async unload(name: string, options?: { space?: string }) {
     const src = this.resolvePath(name);
     this.log.debug('unloading docs from archive at', src);
 
     const objects = await parseArchive(src);
-    this.log.info('deleting', objects.length, 'objects');
-
-    let deleted = 0;
-    let missing = 0;
-
-    await concurrently(20, objects, async (obj) => {
-      try {
-        await this.requester.request({
-          method: 'DELETE',
-          path: options?.space
-            ? uriencode`/s/${options.space}/api/saved_objects/${obj.type}/${obj.id}`
-            : uriencode`/api/saved_objects/${obj.type}/${obj.id}`,
-        });
-        deleted++;
-      } catch (error) {
-        if (isAxiosResponseError(error)) {
-          if (error.response.status === 404) {
-            missing++;
-            return;
-          }
-
-          throw createFailError(
-            `[KbnClientImportExport] ${error.response.status} resp: ${inspect(error.response.data)}`
-          );
-        }
+    this.log.info('deleting', objects.length, 'objects', { space: options?.space });
 
-        throw error;
-      }
-    });
+    const { deleted, missing } = await this.deleteObjects(options?.space, objects);
 
     if (missing) {
       this.log.info(missing, 'saved objects were already deleted');
@@ -144,47 +140,86 @@ export class KbnClientImportExport {
     this.log.success(deleted, 'saved objects deleted');
   }
 
-  async save(name: string, options?: { savedObjectTypes?: string[]; space?: string }) {
+  async save(name: string, options: { types: string[]; space?: string }) {
     const dest = this.resolvePath(name);
     this.log.debug('saving export to', dest);
 
-    let resp;
+    const resp = await this.req(options.space, {
+      method: 'POST',
+      path: '/api/saved_objects/_export',
+      body: {
+        type: options.types,
+        excludeExportDetails: true,
+        includeReferencesDeep: true,
+      },
+    });
+
+    if (typeof resp.data !== 'string') {
+      throw createFailError(`unexpected response from export API: ${inspect(resp.data)}`);
+    }
+
+    const objects = resp.data
+      .split('\n')
+      .filter((l) => !!l)
+      .map((line) => JSON.parse(line));
+
+    const fileContents = objects.map((obj) => JSON.stringify(obj, null, 2)).join('\n\n');
+
+    await Fs.writeFile(dest, fileContents, 'utf-8');
+
+    this.log.success('Exported', objects.length, 'saved objects to', dest);
+  }
+
+  private async req<T>(space: string | undefined, options: ReqOptions) {
+    if (!options.path.startsWith('/')) {
+      throw new Error('options.path must start with a /');
+    }
+
     try {
-      resp = await this.requester.request({
-        method: 'POST',
-        path: options?.space
-          ? uriencode`/s/${options.space}/api/saved_objects/_export`
-          : '/api/saved_objects/_export',
-        body: {
-          type: options?.savedObjectTypes ?? DEFAULT_SAVED_OBJECT_TYPES,
-          excludeExportDetails: true,
-        },
+      return await this.requester.request<T>({
+        ...options,
+        path: space ? uriencode`/s/${space}` + options.path : options.path,
       });
     } catch (error) {
       if (!isAxiosResponseError(error)) {
         throw error;
       }
 
       throw createFailError(
-        `[KbnClientImportExport] ${error.response.status} resp: ${inspect(error.response.data)}`
+        `${error.response.status} resp: ${inspect(error.response.data)}\nreq: ${inspect(
+          error.config
+        )}`
       );
     }
+  }
 
-    if (typeof resp.data !== 'string') {
-      throw createFailError(
-        `[KbnClientImportExport] unexpected response from export API: ${inspect(resp.data)}`
-      );
-    }
+  private async deleteObjects(space: string | undefined, objects: SavedObject[]) {
+    let deleted = 0;
+    let missing = 0;
 
-    const objects = resp.data
-      .split('\n')
-      .filter((l) => !!l)
-      .map((line) => JSON.parse(line));
+    await concurrently(20, objects, async (obj) => {
+      try {
+        await this.requester.request({
+          method: 'DELETE',
+          path: space
+            ? uriencode`/s/${space}/api/saved_objects/${obj.type}/${obj.id}`
+            : uriencode`/api/saved_objects/${obj.type}/${obj.id}`,
+        });
+        deleted++;
+      } catch (error) {
+        if (isAxiosResponseError(error)) {
+          if (error.response.status === 404) {
+            missing++;
+            return;
+          }
 
-    const fileContents = objects.map((obj) => JSON.stringify(obj, null, 2)).join('\n\n');
+          throw createFailError(`${error.response.status} resp: ${inspect(error.response.data)}`);
+        }
 
-    await Fs.writeFile(dest, fileContents, 'utf-8');
+        throw error;
+      }
+    });
 
-    this.log.success('[KbnClientImportExport] Exported', objects.length, 'saved objects to', dest);
+    return { deleted, missing };
   }
 }