diff --git a/docs/developer/telemetry.asciidoc b/docs/developer/telemetry.asciidoc index fe2bf5f957379..c478c091c1c10 100644 --- a/docs/developer/telemetry.asciidoc +++ b/docs/developer/telemetry.asciidoc @@ -8,6 +8,7 @@ The operations we current report timing data for: * Total execution time of `yarn kbn bootstrap`. * Total execution time of `@kbn/optimizer` runs as well as the following metadata about the runs: The number of bundles created, the number of bundles which were cached, usage of `--watch`, `--dist`, `--workers` and `--no-cache` flags, and the count of themes being built. * The time from when you run `yarn start` until both the Kibana server and `@kbn/optimizer` are ready for use. +* The time it takes for the Kibana server to start listening after it is spawned by `yarn start`. Along with the execution time of each execution, we ship the following information about your machine to the service: diff --git a/docs/development/core/server/kibana-plugin-core-server.isavedobjectspointintimefinder.close.md b/docs/development/core/server/kibana-plugin-core-server.isavedobjectspointintimefinder.close.md new file mode 100644 index 0000000000000..f7cfab446eeca --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.isavedobjectspointintimefinder.close.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [ISavedObjectsPointInTimeFinder](./kibana-plugin-core-server.isavedobjectspointintimefinder.md) > [close](./kibana-plugin-core-server.isavedobjectspointintimefinder.close.md) + +## ISavedObjectsPointInTimeFinder.close property + +Closes the Point-In-Time associated with this finder instance. + +Once you have retrieved all of the results you need, it is recommended to call `close()` to clean up the PIT and prevent Elasticsearch from consuming resources unnecessarily. This is only required if you are done iterating and have not yet paged through all of the results: the PIT will automatically be closed for you once you reach the last page of results, or if the underlying call to `find` fails for any reason. + +Signature: + +```typescript +close: () => Promise; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.isavedobjectspointintimefinder.find.md b/docs/development/core/server/kibana-plugin-core-server.isavedobjectspointintimefinder.find.md new file mode 100644 index 0000000000000..1755ff40c2bc0 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.isavedobjectspointintimefinder.find.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [ISavedObjectsPointInTimeFinder](./kibana-plugin-core-server.isavedobjectspointintimefinder.md) > [find](./kibana-plugin-core-server.isavedobjectspointintimefinder.find.md) + +## ISavedObjectsPointInTimeFinder.find property + +An async generator which wraps calls to `savedObjectsClient.find` and iterates over multiple pages of results using `_pit` and `search_after`. This will open a new Point-In-Time (PIT), and continue paging until a set of results is received that's smaller than the designated `perPage` size. + +Signature: + +```typescript +find: () => AsyncGenerator; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.isavedobjectspointintimefinder.md b/docs/development/core/server/kibana-plugin-core-server.isavedobjectspointintimefinder.md new file mode 100644 index 0000000000000..4686df18e0134 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.isavedobjectspointintimefinder.md @@ -0,0 +1,20 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [ISavedObjectsPointInTimeFinder](./kibana-plugin-core-server.isavedobjectspointintimefinder.md) + +## ISavedObjectsPointInTimeFinder interface + + +Signature: + +```typescript +export interface ISavedObjectsPointInTimeFinder +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [close](./kibana-plugin-core-server.isavedobjectspointintimefinder.close.md) | () => Promise<void> | Closes the Point-In-Time associated with this finder instance.Once you have retrieved all of the results you need, it is recommended to call close() to clean up the PIT and prevent Elasticsearch from consuming resources unnecessarily. This is only required if you are done iterating and have not yet paged through all of the results: the PIT will automatically be closed for you once you reach the last page of results, or if the underlying call to find fails for any reason. | +| [find](./kibana-plugin-core-server.isavedobjectspointintimefinder.find.md) | () => AsyncGenerator<SavedObjectsFindResponse> | An async generator which wraps calls to savedObjectsClient.find and iterates over multiple pages of results using _pit and search_after. This will open a new Point-In-Time (PIT), and continue paging until a set of results is received that's smaller than the designated perPage size. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 8dd4667002ead..4bf00d2da6e23 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -98,6 +98,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [IndexSettingsDeprecationInfo](./kibana-plugin-core-server.indexsettingsdeprecationinfo.md) | | | [IRenderOptions](./kibana-plugin-core-server.irenderoptions.md) | | | [IRouter](./kibana-plugin-core-server.irouter.md) | Registers route handlers for specified resource path and method. See [RouteConfig](./kibana-plugin-core-server.routeconfig.md) and [RequestHandler](./kibana-plugin-core-server.requesthandler.md) for more information about arguments to route registrations. | +| [ISavedObjectsPointInTimeFinder](./kibana-plugin-core-server.isavedobjectspointintimefinder.md) | | | [IScopedClusterClient](./kibana-plugin-core-server.iscopedclusterclient.md) | Serves the same purpose as the normal [cluster client](./kibana-plugin-core-server.iclusterclient.md) but exposes an additional asCurrentUser method that doesn't use credentials of the Kibana internal user (as asInternalUser does) to request Elasticsearch API, but rather passes HTTP headers extracted from the current user request to the API instead. | | [IUiSettingsClient](./kibana-plugin-core-server.iuisettingsclient.md) | Server-side client that provides access to the advanced settings stored in elasticsearch. The settings provide control over the behavior of the Kibana application. For example, a user can specify how to display numeric or date fields. Users can adjust the settings via Management UI. | | [KibanaRequestEvents](./kibana-plugin-core-server.kibanarequestevents.md) | Request events. | @@ -158,6 +159,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectsComplexFieldMapping](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.md) | See [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) for documentation. | | [SavedObjectsCoreFieldMapping](./kibana-plugin-core-server.savedobjectscorefieldmapping.md) | See [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) for documentation. | | [SavedObjectsCreateOptions](./kibana-plugin-core-server.savedobjectscreateoptions.md) | | +| [SavedObjectsCreatePointInTimeFinderDependencies](./kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.md) | | | [SavedObjectsDeleteByNamespaceOptions](./kibana-plugin-core-server.savedobjectsdeletebynamespaceoptions.md) | | | [SavedObjectsDeleteFromNamespacesOptions](./kibana-plugin-core-server.savedobjectsdeletefromnamespacesoptions.md) | | | [SavedObjectsDeleteFromNamespacesResponse](./kibana-plugin-core-server.savedobjectsdeletefromnamespacesresponse.md) | | @@ -305,6 +307,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectsClientFactoryProvider](./kibana-plugin-core-server.savedobjectsclientfactoryprovider.md) | Provider to invoke to retrieve a [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md). | | [SavedObjectsClientWrapperFactory](./kibana-plugin-core-server.savedobjectsclientwrapperfactory.md) | Describes the factory used to create instances of Saved Objects Client Wrappers. | | [SavedObjectsClosePointInTimeOptions](./kibana-plugin-core-server.savedobjectsclosepointintimeoptions.md) | | +| [SavedObjectsCreatePointInTimeFinderOptions](./kibana-plugin-core-server.savedobjectscreatepointintimefinderoptions.md) | | | [SavedObjectsExportTransform](./kibana-plugin-core-server.savedobjectsexporttransform.md) | Transformation function used to mutate the exported objects of the associated type.A type's export transform function will be executed once per user-initiated export, for all objects of that type. | | [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) | Describe a [saved object type mapping](./kibana-plugin-core-server.savedobjectstypemappingdefinition.md) field.Please refer to [elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) For the mapping documentation | | [SavedObjectsImportHook](./kibana-plugin-core-server.savedobjectsimporthook.md) | A hook associated with a specific saved object type, that will be invoked during the import process. The hook will have access to the objects of the registered type.Currently, the only supported feature for import hooks is to return warnings to be displayed in the UI when the import succeeds. The only interactions the hook can have with the import process is via the hook's response. Mutating the objects inside the hook's code will have no effect. | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md index dc765260a08ca..79c7d18adf306 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md @@ -6,6 +6,8 @@ Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using [SavedObjectsClient.openPointInTimeForType()](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md). +Only use this API if you have an advanced use case that's not solved by the [SavedObjectsClient.createPointInTimeFinder()](./kibana-plugin-core-server.savedobjectsclient.createpointintimefinder.md) method. + Signature: ```typescript diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.createpointintimefinder.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.createpointintimefinder.md new file mode 100644 index 0000000000000..8afd963464574 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.createpointintimefinder.md @@ -0,0 +1,53 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) > [createPointInTimeFinder](./kibana-plugin-core-server.savedobjectsclient.createpointintimefinder.md) + +## SavedObjectsClient.createPointInTimeFinder() method + +Returns a [ISavedObjectsPointInTimeFinder](./kibana-plugin-core-server.isavedobjectspointintimefinder.md) to help page through large sets of saved objects. We strongly recommend using this API for any `find` queries that might return more than 1000 saved objects, however this API is only intended for use in server-side "batch" processing of objects where you are collecting all objects in memory or streaming them back to the client. + +Do NOT use this API in a route handler to facilitate paging through saved objects on the client-side unless you are streaming all of the results back to the client at once. Because the returned generator is stateful, you cannot rely on subsequent http requests retrieving new pages from the same Kibana server in multi-instance deployments. + +The generator wraps calls to [SavedObjectsClient.find()](./kibana-plugin-core-server.savedobjectsclient.find.md) and iterates over multiple pages of results using `_pit` and `search_after`. This will open a new Point-In-Time (PIT), and continue paging until a set of results is received that's smaller than the designated `perPage`. + +Once you have retrieved all of the results you need, it is recommended to call `close()` to clean up the PIT and prevent Elasticsearch from consuming resources unnecessarily. This is only required if you are done iterating and have not yet paged through all of the results: the PIT will automatically be closed for you once you reach the last page of results, or if the underlying call to `find` fails for any reason. + +Signature: + +```typescript +createPointInTimeFinder(findOptions: SavedObjectsCreatePointInTimeFinderOptions, dependencies?: SavedObjectsCreatePointInTimeFinderDependencies): ISavedObjectsPointInTimeFinder; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| findOptions | SavedObjectsCreatePointInTimeFinderOptions | | +| dependencies | SavedObjectsCreatePointInTimeFinderDependencies | | + +Returns: + +`ISavedObjectsPointInTimeFinder` + +## Example + + +```ts +const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { + type: 'visualization', + search: 'foo*', + perPage: 100, +}; + +const finder = savedObjectsClient.createPointInTimeFinder(findOptions); + +const responses: SavedObjectFindResponse[] = []; +for await (const response of finder.find()) { + responses.push(...response); + if (doneSearching) { + await finder.close(); + } +} + +``` + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md index 887f7f7d93a87..95c2251f72c90 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md @@ -30,13 +30,14 @@ The constructor for this class is marked as internal. Third-party code should no | [bulkGet(objects, options)](./kibana-plugin-core-server.savedobjectsclient.bulkget.md) | | Returns an array of objects by id | | [bulkUpdate(objects, options)](./kibana-plugin-core-server.savedobjectsclient.bulkupdate.md) | | Bulk Updates multiple SavedObject at once | | [checkConflicts(objects, options)](./kibana-plugin-core-server.savedobjectsclient.checkconflicts.md) | | Check what conflicts will result when creating a given array of saved objects. This includes "unresolvable conflicts", which are multi-namespace objects that exist in a different namespace; such conflicts cannot be resolved/overwritten. | -| [closePointInTime(id, options)](./kibana-plugin-core-server.savedobjectsclient.closepointintime.md) | | Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using [SavedObjectsClient.openPointInTimeForType()](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md). | +| [closePointInTime(id, options)](./kibana-plugin-core-server.savedobjectsclient.closepointintime.md) | | Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using [SavedObjectsClient.openPointInTimeForType()](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md).Only use this API if you have an advanced use case that's not solved by the [SavedObjectsClient.createPointInTimeFinder()](./kibana-plugin-core-server.savedobjectsclient.createpointintimefinder.md) method. | | [create(type, attributes, options)](./kibana-plugin-core-server.savedobjectsclient.create.md) | | Persists a SavedObject | +| [createPointInTimeFinder(findOptions, dependencies)](./kibana-plugin-core-server.savedobjectsclient.createpointintimefinder.md) | | Returns a [ISavedObjectsPointInTimeFinder](./kibana-plugin-core-server.isavedobjectspointintimefinder.md) to help page through large sets of saved objects. We strongly recommend using this API for any find queries that might return more than 1000 saved objects, however this API is only intended for use in server-side "batch" processing of objects where you are collecting all objects in memory or streaming them back to the client.Do NOT use this API in a route handler to facilitate paging through saved objects on the client-side unless you are streaming all of the results back to the client at once. Because the returned generator is stateful, you cannot rely on subsequent http requests retrieving new pages from the same Kibana server in multi-instance deployments.The generator wraps calls to [SavedObjectsClient.find()](./kibana-plugin-core-server.savedobjectsclient.find.md) and iterates over multiple pages of results using _pit and search_after. This will open a new Point-In-Time (PIT), and continue paging until a set of results is received that's smaller than the designated perPage.Once you have retrieved all of the results you need, it is recommended to call close() to clean up the PIT and prevent Elasticsearch from consuming resources unnecessarily. This is only required if you are done iterating and have not yet paged through all of the results: the PIT will automatically be closed for you once you reach the last page of results, or if the underlying call to find fails for any reason. | | [delete(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.delete.md) | | Deletes a SavedObject | | [deleteFromNamespaces(type, id, namespaces, options)](./kibana-plugin-core-server.savedobjectsclient.deletefromnamespaces.md) | | Removes namespaces from a SavedObject | | [find(options)](./kibana-plugin-core-server.savedobjectsclient.find.md) | | Find all SavedObjects matching the search query | | [get(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.get.md) | | Retrieves a single object | -| [openPointInTimeForType(type, options)](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md) | | Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned id can then be passed to [SavedObjectsClient.find()](./kibana-plugin-core-server.savedobjectsclient.find.md) to search against that PIT. | +| [openPointInTimeForType(type, options)](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md) | | Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned id can then be passed to [SavedObjectsClient.find()](./kibana-plugin-core-server.savedobjectsclient.find.md) to search against that PIT.Only use this API if you have an advanced use case that's not solved by the [SavedObjectsClient.createPointInTimeFinder()](./kibana-plugin-core-server.savedobjectsclient.createpointintimefinder.md) method. | | [removeReferencesTo(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.removereferencesto.md) | | Updates all objects containing a reference to the given {type, id} tuple to remove the said reference. | | [resolve(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.resolve.md) | | Resolves a single object, using any legacy URL alias if it exists | | [update(type, id, attributes, options)](./kibana-plugin-core-server.savedobjectsclient.update.md) | | Updates an SavedObject | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md index 56c1d6d1ddc33..c76159ffa5032 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md @@ -6,6 +6,8 @@ Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned `id` can then be passed to [SavedObjectsClient.find()](./kibana-plugin-core-server.savedobjectsclient.find.md) to search against that PIT. +Only use this API if you have an advanced use case that's not solved by the [SavedObjectsClient.createPointInTimeFinder()](./kibana-plugin-core-server.savedobjectsclient.createpointintimefinder.md) method. + Signature: ```typescript diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.client.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.client.md new file mode 100644 index 0000000000000..95ab9e225c049 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.client.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsCreatePointInTimeFinderDependencies](./kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.md) > [client](./kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.client.md) + +## SavedObjectsCreatePointInTimeFinderDependencies.client property + +Signature: + +```typescript +client: Pick; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.md new file mode 100644 index 0000000000000..47c640bfabcb0 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.md @@ -0,0 +1,19 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsCreatePointInTimeFinderDependencies](./kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.md) + +## SavedObjectsCreatePointInTimeFinderDependencies interface + + +Signature: + +```typescript +export interface SavedObjectsCreatePointInTimeFinderDependencies +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [client](./kibana-plugin-core-server.savedobjectscreatepointintimefinderdependencies.client.md) | Pick<SavedObjectsClientContract, 'find' | 'openPointInTimeForType' | 'closePointInTime'> | | + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectscreatepointintimefinderoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreatepointintimefinderoptions.md new file mode 100644 index 0000000000000..928c6f72bcbf5 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreatepointintimefinderoptions.md @@ -0,0 +1,12 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsCreatePointInTimeFinderOptions](./kibana-plugin-core-server.savedobjectscreatepointintimefinderoptions.md) + +## SavedObjectsCreatePointInTimeFinderOptions type + + +Signature: + +```typescript +export declare type SavedObjectsCreatePointInTimeFinderOptions = Omit; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md index 8f9dca35fa362..b9d81c89bffd7 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md @@ -6,6 +6,8 @@ Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using `openPointInTimeForType`. +Only use this API if you have an advanced use case that's not solved by the [SavedObjectsRepository.createPointInTimeFinder()](./kibana-plugin-core-server.savedobjectsrepository.createpointintimefinder.md) method. + Signature: ```typescript diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.createpointintimefinder.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.createpointintimefinder.md new file mode 100644 index 0000000000000..5d9d2857f6e0b --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.createpointintimefinder.md @@ -0,0 +1,53 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsRepository](./kibana-plugin-core-server.savedobjectsrepository.md) > [createPointInTimeFinder](./kibana-plugin-core-server.savedobjectsrepository.createpointintimefinder.md) + +## SavedObjectsRepository.createPointInTimeFinder() method + +Returns a [ISavedObjectsPointInTimeFinder](./kibana-plugin-core-server.isavedobjectspointintimefinder.md) to help page through large sets of saved objects. We strongly recommend using this API for any `find` queries that might return more than 1000 saved objects, however this API is only intended for use in server-side "batch" processing of objects where you are collecting all objects in memory or streaming them back to the client. + +Do NOT use this API in a route handler to facilitate paging through saved objects on the client-side unless you are streaming all of the results back to the client at once. Because the returned generator is stateful, you cannot rely on subsequent http requests retrieving new pages from the same Kibana server in multi-instance deployments. + +This generator wraps calls to [SavedObjectsRepository.find()](./kibana-plugin-core-server.savedobjectsrepository.find.md) and iterates over multiple pages of results using `_pit` and `search_after`. This will open a new Point-In-Time (PIT), and continue paging until a set of results is received that's smaller than the designated `perPage`. + +Once you have retrieved all of the results you need, it is recommended to call `close()` to clean up the PIT and prevent Elasticsearch from consuming resources unnecessarily. This is only required if you are done iterating and have not yet paged through all of the results: the PIT will automatically be closed for you once you reach the last page of results, or if the underlying call to `find` fails for any reason. + +Signature: + +```typescript +createPointInTimeFinder(findOptions: SavedObjectsCreatePointInTimeFinderOptions, dependencies?: SavedObjectsCreatePointInTimeFinderDependencies): ISavedObjectsPointInTimeFinder; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| findOptions | SavedObjectsCreatePointInTimeFinderOptions | | +| dependencies | SavedObjectsCreatePointInTimeFinderDependencies | | + +Returns: + +`ISavedObjectsPointInTimeFinder` + +## Example + + +```ts +const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { + type: 'visualization', + search: 'foo*', + perPage: 100, +}; + +const finder = savedObjectsClient.createPointInTimeFinder(findOptions); + +const responses: SavedObjectFindResponse[] = []; +for await (const response of finder.find()) { + responses.push(...response); + if (doneSearching) { + await finder.close(); + } +} + +``` + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md index 632d9c279cb88..00e6ed3aeddfc 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md @@ -20,15 +20,16 @@ export declare class SavedObjectsRepository | [bulkGet(objects, options)](./kibana-plugin-core-server.savedobjectsrepository.bulkget.md) | | Returns an array of objects by id | | [bulkUpdate(objects, options)](./kibana-plugin-core-server.savedobjectsrepository.bulkupdate.md) | | Updates multiple objects in bulk | | [checkConflicts(objects, options)](./kibana-plugin-core-server.savedobjectsrepository.checkconflicts.md) | | Check what conflicts will result when creating a given array of saved objects. This includes "unresolvable conflicts", which are multi-namespace objects that exist in a different namespace; such conflicts cannot be resolved/overwritten. | -| [closePointInTime(id, options)](./kibana-plugin-core-server.savedobjectsrepository.closepointintime.md) | | Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using openPointInTimeForType. | +| [closePointInTime(id, options)](./kibana-plugin-core-server.savedobjectsrepository.closepointintime.md) | | Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using openPointInTimeForType.Only use this API if you have an advanced use case that's not solved by the [SavedObjectsRepository.createPointInTimeFinder()](./kibana-plugin-core-server.savedobjectsrepository.createpointintimefinder.md) method. | | [create(type, attributes, options)](./kibana-plugin-core-server.savedobjectsrepository.create.md) | | Persists an object | +| [createPointInTimeFinder(findOptions, dependencies)](./kibana-plugin-core-server.savedobjectsrepository.createpointintimefinder.md) | | Returns a [ISavedObjectsPointInTimeFinder](./kibana-plugin-core-server.isavedobjectspointintimefinder.md) to help page through large sets of saved objects. We strongly recommend using this API for any find queries that might return more than 1000 saved objects, however this API is only intended for use in server-side "batch" processing of objects where you are collecting all objects in memory or streaming them back to the client.Do NOT use this API in a route handler to facilitate paging through saved objects on the client-side unless you are streaming all of the results back to the client at once. Because the returned generator is stateful, you cannot rely on subsequent http requests retrieving new pages from the same Kibana server in multi-instance deployments.This generator wraps calls to [SavedObjectsRepository.find()](./kibana-plugin-core-server.savedobjectsrepository.find.md) and iterates over multiple pages of results using _pit and search_after. This will open a new Point-In-Time (PIT), and continue paging until a set of results is received that's smaller than the designated perPage.Once you have retrieved all of the results you need, it is recommended to call close() to clean up the PIT and prevent Elasticsearch from consuming resources unnecessarily. This is only required if you are done iterating and have not yet paged through all of the results: the PIT will automatically be closed for you once you reach the last page of results, or if the underlying call to find fails for any reason. | | [delete(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.delete.md) | | Deletes an object | | [deleteByNamespace(namespace, options)](./kibana-plugin-core-server.savedobjectsrepository.deletebynamespace.md) | | Deletes all objects from the provided namespace. | | [deleteFromNamespaces(type, id, namespaces, options)](./kibana-plugin-core-server.savedobjectsrepository.deletefromnamespaces.md) | | Removes one or more namespaces from a given multi-namespace saved object. If no namespaces remain, the saved object is deleted entirely. This method and \[addToNamespaces\][SavedObjectsRepository.addToNamespaces()](./kibana-plugin-core-server.savedobjectsrepository.addtonamespaces.md) are the only ways to change which Spaces a multi-namespace saved object is shared to. | | [find(options)](./kibana-plugin-core-server.savedobjectsrepository.find.md) | | | | [get(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.get.md) | | Gets a single object | | [incrementCounter(type, id, counterFields, options)](./kibana-plugin-core-server.savedobjectsrepository.incrementcounter.md) | | Increments all the specified counter fields (by one by default). Creates the document if one doesn't exist for the given id. | -| [openPointInTimeForType(type, { keepAlive, preference })](./kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md) | | Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned id can then be passed to SavedObjects.find to search against that PIT. | +| [openPointInTimeForType(type, { keepAlive, preference })](./kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md) | | Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned id can then be passed to SavedObjects.find to search against that PIT.Only use this API if you have an advanced use case that's not solved by the [SavedObjectsRepository.createPointInTimeFinder()](./kibana-plugin-core-server.savedobjectsrepository.createpointintimefinder.md) method. | | [removeReferencesTo(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.removereferencesto.md) | | Updates all objects containing a reference to the given {type, id} tuple to remove the said reference. | | [resolve(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.resolve.md) | | Resolves a single object, using any legacy URL alias if it exists | | [update(type, id, attributes, options)](./kibana-plugin-core-server.savedobjectsrepository.update.md) | | Updates an object | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md index 6b66882484520..b33765bb79dd8 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md @@ -6,6 +6,8 @@ Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. +Only use this API if you have an advanced use case that's not solved by the [SavedObjectsRepository.createPointInTimeFinder()](./kibana-plugin-core-server.savedobjectsrepository.createpointintimefinder.md) method. + Signature: ```typescript diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.start.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.start.md index f479ffd52e9b8..025cab9f48c1a 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.start.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.start.md @@ -12,7 +12,7 @@ start(core: CoreStart): { fieldFormatServiceFactory: (uiSettings: import("../../../core/server").IUiSettingsClient) => Promise; }; indexPatterns: { - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: import("../../../core/server").ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: import("../../../core/server").ElasticsearchClient) => Promise; }; search: ISearchStart>; }; @@ -31,7 +31,7 @@ start(core: CoreStart): { fieldFormatServiceFactory: (uiSettings: import("../../../core/server").IUiSettingsClient) => Promise; }; indexPatterns: { - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: import("../../../core/server").ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: import("../../../core/server").ElasticsearchClient) => Promise; }; search: ISearchStart>; }` diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 0694bc4ffdb0f..d82b7b83e8f15 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -13,8 +13,8 @@ import Joi from 'joi'; // valid pattern for ID // enforced camel-case identifiers for consistency const ID_PATTERN = /^[a-zA-Z0-9_]+$/; -const INSPECTING = - process.execArgv.includes('--inspect') || process.execArgv.includes('--inspect-brk'); +// it will search both --inspect and --inspect-brk +const INSPECTING = !!process.execArgv.find((arg) => arg.includes('--inspect')); const urlPartsSchema = () => Joi.object() diff --git a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js index 4abbc3d29fe7c..a43d3a09c7d70 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js +++ b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.js @@ -62,15 +62,11 @@ function collectCliArgs(config, { installDir, extraKbnOpts }) { const buildArgs = config.get('kbnTestServer.buildArgs') || []; const sourceArgs = config.get('kbnTestServer.sourceArgs') || []; const serverArgs = config.get('kbnTestServer.serverArgs') || []; - const execArgv = process.execArgv || []; return pipe( serverArgs, (args) => (installDir ? args.filter((a) => a !== '--oss') : args), - (args) => - installDir - ? [...buildArgs, ...args] - : [...execArgv, KIBANA_EXEC_PATH, ...sourceArgs, ...args], + (args) => (installDir ? [...buildArgs, ...args] : [KIBANA_EXEC_PATH, ...sourceArgs, ...args]), (args) => args.concat(extraKbnOpts || []) ); } diff --git a/src/core/server/http/cookie_session_storage.test.ts b/src/core/server/http/cookie_session_storage.test.ts index f00cbb928d631..c802163866423 100644 --- a/src/core/server/http/cookie_session_storage.test.ts +++ b/src/core/server/http/cookie_session_storage.test.ts @@ -124,7 +124,9 @@ const cookieOptions = { path, }; -describe('Cookie based SessionStorage', () => { +// FLAKY: https://github.com/elastic/kibana/issues/89318 +// https://github.com/elastic/kibana/issues/89319 +describe.skip('Cookie based SessionStorage', () => { describe('#set()', () => { it('Should write to session storage & set cookies', async () => { const { server: innerServer, createRouter } = await server.setup(setupDeps); diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 3e336dceb83d7..788c179501a80 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -282,6 +282,9 @@ export type { SavedObjectsClientFactoryProvider, SavedObjectsClosePointInTimeOptions, SavedObjectsClosePointInTimeResponse, + ISavedObjectsPointInTimeFinder, + SavedObjectsCreatePointInTimeFinderDependencies, + SavedObjectsCreatePointInTimeFinderOptions, SavedObjectsCreateOptions, SavedObjectsExportResultDetails, SavedObjectsFindResult, diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.ts b/src/core/server/saved_objects/export/saved_objects_exporter.ts index c1c0ea73f0bd3..868efa872d643 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.ts @@ -9,7 +9,7 @@ import { createListStream } from '@kbn/utils'; import { PublicMethodsOf } from '@kbn/utility-types'; import { Logger } from '../../logging'; -import { SavedObject, SavedObjectsClientContract, SavedObjectsFindOptions } from '../types'; +import { SavedObject, SavedObjectsClientContract } from '../types'; import { SavedObjectsFindResult } from '../service'; import { ISavedObjectTypeRegistry } from '../saved_objects_type_registry'; import { fetchNestedDependencies } from './fetch_nested_dependencies'; @@ -23,7 +23,6 @@ import { } from './types'; import { SavedObjectsExportError } from './errors'; import { applyExportTransforms } from './apply_export_transforms'; -import { createPointInTimeFinder } from './point_in_time_finder'; import { byIdAscComparator, getPreservedOrderComparator, SavedObjectComparator } from './utils'; /** @@ -168,18 +167,12 @@ export class SavedObjectsExporter { hasReference, search, }: SavedObjectsExportByTypeOptions) { - const findOptions: SavedObjectsFindOptions = { + const finder = this.#savedObjectsClient.createPointInTimeFinder({ type: types, hasReference, hasReferenceOperator: hasReference ? 'OR' : undefined, search, namespaces: namespace ? [namespace] : undefined, - }; - - const finder = createPointInTimeFinder({ - findOptions, - logger: this.#log, - savedObjectsClient: this.#savedObjectsClient, }); const hits: SavedObjectsFindResult[] = []; diff --git a/src/core/server/saved_objects/saved_objects_service.test.ts b/src/core/server/saved_objects/saved_objects_service.test.ts index d589809e38f01..52f8dcd310509 100644 --- a/src/core/server/saved_objects/saved_objects_service.test.ts +++ b/src/core/server/saved_objects/saved_objects_service.test.ts @@ -274,7 +274,7 @@ describe('SavedObjectsService', () => { expect(coreStart.elasticsearch.client.asScoped).toHaveBeenCalledWith(req); const [ - [, , , , includedHiddenTypes], + [, , , , , includedHiddenTypes], ] = (SavedObjectsRepository.createRepository as jest.Mocked).mock.calls; expect(includedHiddenTypes).toEqual([]); @@ -292,7 +292,7 @@ describe('SavedObjectsService', () => { createScopedRepository(req, ['someHiddenType']); const [ - [, , , , includedHiddenTypes], + [, , , , , includedHiddenTypes], ] = (SavedObjectsRepository.createRepository as jest.Mocked).mock.calls; expect(includedHiddenTypes).toEqual(['someHiddenType']); @@ -311,7 +311,7 @@ describe('SavedObjectsService', () => { createInternalRepository(); const [ - [, , , client, includedHiddenTypes], + [, , , client, , includedHiddenTypes], ] = (SavedObjectsRepository.createRepository as jest.Mocked).mock.calls; expect(coreStart.elasticsearch.client.asInternalUser).toBe(client); @@ -328,7 +328,7 @@ describe('SavedObjectsService', () => { createInternalRepository(['someHiddenType']); const [ - [, , , , includedHiddenTypes], + [, , , , , includedHiddenTypes], ] = (SavedObjectsRepository.createRepository as jest.Mocked).mock.calls; expect(includedHiddenTypes).toEqual(['someHiddenType']); diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index fce7f12384456..8e4320eb841f8 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -421,6 +421,7 @@ export class SavedObjectsService this.typeRegistry, kibanaConfig.index, esClient, + this.logger.get('repository'), includedHiddenTypes ); }; diff --git a/src/core/server/saved_objects/service/index.ts b/src/core/server/saved_objects/service/index.ts index 1186e15cbef4a..8a66e6176d1f5 100644 --- a/src/core/server/saved_objects/service/index.ts +++ b/src/core/server/saved_objects/service/index.ts @@ -8,6 +8,9 @@ export { SavedObjectsErrorHelpers, SavedObjectsClientProvider, SavedObjectsUtils } from './lib'; export type { SavedObjectsRepository, + ISavedObjectsPointInTimeFinder, + SavedObjectsCreatePointInTimeFinderOptions, + SavedObjectsCreatePointInTimeFinderDependencies, ISavedObjectsClientProvider, SavedObjectsClientProviderOptions, SavedObjectsClientWrapperFactory, diff --git a/src/core/server/saved_objects/service/lib/index.ts b/src/core/server/saved_objects/service/lib/index.ts index d05552bc6e55e..09bce81b14c39 100644 --- a/src/core/server/saved_objects/service/lib/index.ts +++ b/src/core/server/saved_objects/service/lib/index.ts @@ -8,6 +8,13 @@ export type { ISavedObjectsRepository, SavedObjectsRepository } from './repository'; export { SavedObjectsClientProvider } from './scoped_client_provider'; + +export type { + ISavedObjectsPointInTimeFinder, + SavedObjectsCreatePointInTimeFinderOptions, + SavedObjectsCreatePointInTimeFinderDependencies, +} from './point_in_time_finder'; + export type { SavedObjectsClientWrapperFactory, SavedObjectsClientWrapperOptions, diff --git a/src/core/server/saved_objects/service/lib/point_in_time_finder.mock.ts b/src/core/server/saved_objects/service/lib/point_in_time_finder.mock.ts new file mode 100644 index 0000000000000..c689eb319898b --- /dev/null +++ b/src/core/server/saved_objects/service/lib/point_in_time_finder.mock.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { loggerMock, MockedLogger } from '../../../logging/logger.mock'; +import type { SavedObjectsClientContract } from '../../types'; +import type { ISavedObjectsRepository } from './repository'; +import { PointInTimeFinder } from './point_in_time_finder'; + +const createPointInTimeFinderMock = ({ + logger = loggerMock.create(), + savedObjectsMock, +}: { + logger?: MockedLogger; + savedObjectsMock: jest.Mocked; +}): jest.Mock => { + const mock = jest.fn(); + + // To simplify testing, we use the actual implementation here, but pass through the + // mocked dependencies. This allows users to set their own `mockResolvedValue` on + // the SO client mock and have it reflected when using `createPointInTimeFinder`. + mock.mockImplementation((findOptions) => { + const finder = new PointInTimeFinder(findOptions, { + logger, + client: savedObjectsMock, + }); + + jest.spyOn(finder, 'find'); + jest.spyOn(finder, 'close'); + + return finder; + }); + + return mock; +}; + +export const savedObjectsPointInTimeFinderMock = { + create: createPointInTimeFinderMock, +}; diff --git a/src/core/server/saved_objects/export/point_in_time_finder.test.ts b/src/core/server/saved_objects/service/lib/point_in_time_finder.test.ts similarity index 57% rename from src/core/server/saved_objects/export/point_in_time_finder.test.ts rename to src/core/server/saved_objects/service/lib/point_in_time_finder.test.ts index cd79c7a4b81e5..044bb45269538 100644 --- a/src/core/server/saved_objects/export/point_in_time_finder.test.ts +++ b/src/core/server/saved_objects/service/lib/point_in_time_finder.test.ts @@ -6,12 +6,15 @@ * Side Public License, v 1. */ -import { savedObjectsClientMock } from '../service/saved_objects_client.mock'; -import { loggerMock, MockedLogger } from '../../logging/logger.mock'; -import { SavedObjectsFindOptions } from '../types'; -import { SavedObjectsFindResult } from '../service'; +import { loggerMock, MockedLogger } from '../../../logging/logger.mock'; +import type { SavedObjectsClientContract } from '../../types'; +import type { SavedObjectsFindResult } from '../'; +import { savedObjectsRepositoryMock } from './repository.mock'; -import { createPointInTimeFinder } from './point_in_time_finder'; +import { + PointInTimeFinder, + SavedObjectsCreatePointInTimeFinderOptions, +} from './point_in_time_finder'; const mockHits = [ { @@ -40,26 +43,31 @@ const mockHits = [ describe('createPointInTimeFinder()', () => { let logger: MockedLogger; - let savedObjectsClient: ReturnType; + let find: jest.Mocked['find']; + let openPointInTimeForType: jest.Mocked['openPointInTimeForType']; + let closePointInTime: jest.Mocked['closePointInTime']; beforeEach(() => { logger = loggerMock.create(); - savedObjectsClient = savedObjectsClientMock.create(); + const mockRepository = savedObjectsRepositoryMock.create(); + find = mockRepository.find; + openPointInTimeForType = mockRepository.openPointInTimeForType; + closePointInTime = mockRepository.closePointInTime; }); describe('#find', () => { test('throws if a PIT is already open', async () => { - savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + openPointInTimeForType.mockResolvedValueOnce({ id: 'abc123', }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: mockHits, pit_id: 'abc123', per_page: 1, page: 0, }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: mockHits, pit_id: 'abc123', @@ -67,31 +75,38 @@ describe('createPointInTimeFinder()', () => { page: 1, }); - const findOptions: SavedObjectsFindOptions = { + const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { type: ['visualization'], search: 'foo*', perPage: 1, }; - const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); + const finder = new PointInTimeFinder(findOptions, { + logger, + client: { + find, + openPointInTimeForType, + closePointInTime, + }, + }); await finder.find().next(); - expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); - savedObjectsClient.find.mockClear(); + expect(find).toHaveBeenCalledTimes(1); + find.mockClear(); expect(async () => { await finder.find().next(); }).rejects.toThrowErrorMatchingInlineSnapshot( `"Point In Time has already been opened for this finder instance. Please call \`close()\` before calling \`find()\` again."` ); - expect(savedObjectsClient.find).toHaveBeenCalledTimes(0); + expect(find).toHaveBeenCalledTimes(0); }); test('works with a single page of results', async () => { - savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + openPointInTimeForType.mockResolvedValueOnce({ id: 'abc123', }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: mockHits, pit_id: 'abc123', @@ -99,22 +114,29 @@ describe('createPointInTimeFinder()', () => { page: 0, }); - const findOptions: SavedObjectsFindOptions = { + const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { type: ['visualization'], search: 'foo*', }; - const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); + const finder = new PointInTimeFinder(findOptions, { + logger, + client: { + find, + openPointInTimeForType, + closePointInTime, + }, + }); const hits: SavedObjectsFindResult[] = []; for await (const result of finder.find()) { hits.push(...result.saved_objects); } expect(hits.length).toBe(2); - expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); - expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1); - expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); - expect(savedObjectsClient.find).toHaveBeenCalledWith( + expect(openPointInTimeForType).toHaveBeenCalledTimes(1); + expect(closePointInTime).toHaveBeenCalledTimes(1); + expect(find).toHaveBeenCalledTimes(1); + expect(find).toHaveBeenCalledWith( expect.objectContaining({ pit: expect.objectContaining({ id: 'abc123', keepAlive: '2m' }), sortField: 'updated_at', @@ -125,24 +147,24 @@ describe('createPointInTimeFinder()', () => { }); test('works with multiple pages of results', async () => { - savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + openPointInTimeForType.mockResolvedValueOnce({ id: 'abc123', }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: [mockHits[0]], pit_id: 'abc123', per_page: 1, page: 0, }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: [mockHits[1]], pit_id: 'abc123', per_page: 1, page: 0, }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: [], per_page: 1, @@ -150,25 +172,32 @@ describe('createPointInTimeFinder()', () => { page: 0, }); - const findOptions: SavedObjectsFindOptions = { + const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { type: ['visualization'], search: 'foo*', perPage: 1, }; - const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); + const finder = new PointInTimeFinder(findOptions, { + logger, + client: { + find, + openPointInTimeForType, + closePointInTime, + }, + }); const hits: SavedObjectsFindResult[] = []; for await (const result of finder.find()) { hits.push(...result.saved_objects); } expect(hits.length).toBe(2); - expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); - expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1); + expect(openPointInTimeForType).toHaveBeenCalledTimes(1); + expect(closePointInTime).toHaveBeenCalledTimes(1); // called 3 times since we need a 3rd request to check if we // are done paginating through results. - expect(savedObjectsClient.find).toHaveBeenCalledTimes(3); - expect(savedObjectsClient.find).toHaveBeenCalledWith( + expect(find).toHaveBeenCalledTimes(3); + expect(find).toHaveBeenCalledWith( expect.objectContaining({ pit: expect.objectContaining({ id: 'abc123', keepAlive: '2m' }), sortField: 'updated_at', @@ -181,10 +210,10 @@ describe('createPointInTimeFinder()', () => { describe('#close', () => { test('calls closePointInTime with correct ID', async () => { - savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + openPointInTimeForType.mockResolvedValueOnce({ id: 'test', }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 1, saved_objects: [mockHits[0]], pit_id: 'test', @@ -192,41 +221,48 @@ describe('createPointInTimeFinder()', () => { page: 0, }); - const findOptions: SavedObjectsFindOptions = { + const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { type: ['visualization'], search: 'foo*', perPage: 2, }; - const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); + const finder = new PointInTimeFinder(findOptions, { + logger, + client: { + find, + openPointInTimeForType, + closePointInTime, + }, + }); const hits: SavedObjectsFindResult[] = []; for await (const result of finder.find()) { hits.push(...result.saved_objects); await finder.close(); } - expect(savedObjectsClient.closePointInTime).toHaveBeenCalledWith('test'); + expect(closePointInTime).toHaveBeenCalledWith('test'); }); test('causes generator to stop', async () => { - savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + openPointInTimeForType.mockResolvedValueOnce({ id: 'test', }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: [mockHits[0]], pit_id: 'test', per_page: 1, page: 0, }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: [mockHits[1]], pit_id: 'test', per_page: 1, page: 0, }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: [], per_page: 1, @@ -234,36 +270,50 @@ describe('createPointInTimeFinder()', () => { page: 0, }); - const findOptions: SavedObjectsFindOptions = { + const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { type: ['visualization'], search: 'foo*', perPage: 1, }; - const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); + const finder = new PointInTimeFinder(findOptions, { + logger, + client: { + find, + openPointInTimeForType, + closePointInTime, + }, + }); const hits: SavedObjectsFindResult[] = []; for await (const result of finder.find()) { hits.push(...result.saved_objects); await finder.close(); } - expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1); + expect(closePointInTime).toHaveBeenCalledTimes(1); expect(hits.length).toBe(1); }); test('is called if `find` throws an error', async () => { - savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + openPointInTimeForType.mockResolvedValueOnce({ id: 'test', }); - savedObjectsClient.find.mockRejectedValueOnce(new Error('oops')); + find.mockRejectedValueOnce(new Error('oops')); - const findOptions: SavedObjectsFindOptions = { + const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { type: ['visualization'], search: 'foo*', perPage: 2, }; - const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); + const finder = new PointInTimeFinder(findOptions, { + logger, + client: { + find, + openPointInTimeForType, + closePointInTime, + }, + }); const hits: SavedObjectsFindResult[] = []; try { for await (const result of finder.find()) { @@ -273,21 +323,21 @@ describe('createPointInTimeFinder()', () => { // intentionally empty } - expect(savedObjectsClient.closePointInTime).toHaveBeenCalledWith('test'); + expect(closePointInTime).toHaveBeenCalledWith('test'); }); test('finder can be reused after closing', async () => { - savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + openPointInTimeForType.mockResolvedValueOnce({ id: 'abc123', }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: mockHits, pit_id: 'abc123', per_page: 1, page: 0, }); - savedObjectsClient.find.mockResolvedValueOnce({ + find.mockResolvedValueOnce({ total: 2, saved_objects: mockHits, pit_id: 'abc123', @@ -295,13 +345,20 @@ describe('createPointInTimeFinder()', () => { page: 1, }); - const findOptions: SavedObjectsFindOptions = { + const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { type: ['visualization'], search: 'foo*', perPage: 1, }; - const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); + const finder = new PointInTimeFinder(findOptions, { + logger, + client: { + find, + openPointInTimeForType, + closePointInTime, + }, + }); const findA = finder.find(); await findA.next(); @@ -313,9 +370,9 @@ describe('createPointInTimeFinder()', () => { expect((await findA.next()).done).toBe(true); expect((await findB.next()).done).toBe(true); - expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(2); - expect(savedObjectsClient.find).toHaveBeenCalledTimes(2); - expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(2); + expect(openPointInTimeForType).toHaveBeenCalledTimes(2); + expect(find).toHaveBeenCalledTimes(2); + expect(closePointInTime).toHaveBeenCalledTimes(2); }); }); }); diff --git a/src/core/server/saved_objects/export/point_in_time_finder.ts b/src/core/server/saved_objects/service/lib/point_in_time_finder.ts similarity index 58% rename from src/core/server/saved_objects/export/point_in_time_finder.ts rename to src/core/server/saved_objects/service/lib/point_in_time_finder.ts index a4575fefe3c6e..9a8dcceafebb2 100644 --- a/src/core/server/saved_objects/export/point_in_time_finder.ts +++ b/src/core/server/saved_objects/service/lib/point_in_time_finder.ts @@ -6,79 +6,76 @@ * Side Public License, v 1. */ import type { estypes } from '@elastic/elasticsearch'; -import { Logger } from '../../logging'; -import { SavedObjectsClientContract, SavedObjectsFindOptions } from '../types'; -import { SavedObjectsFindResponse } from '../service'; +import type { Logger } from '../../../logging'; +import type { SavedObjectsFindOptions, SavedObjectsClientContract } from '../../types'; +import type { SavedObjectsFindResponse } from '../'; + +type PointInTimeFinderClient = Pick< + SavedObjectsClientContract, + 'find' | 'openPointInTimeForType' | 'closePointInTime' +>; + +/** + * @public + */ +export type SavedObjectsCreatePointInTimeFinderOptions = Omit< + SavedObjectsFindOptions, + 'page' | 'pit' | 'searchAfter' +>; /** - * Returns a generator to help page through large sets of saved objects. - * - * The generator wraps calls to `SavedObjects.find` and iterates over - * multiple pages of results using `_pit` and `search_after`. This will - * open a new Point In Time (PIT), and continue paging until a set of - * results is received that's smaller than the designated `perPage`. - * - * Once you have retrieved all of the results you need, it is recommended - * to call `close()` to clean up the PIT and prevent Elasticsearch from - * consuming resources unnecessarily. This will automatically be done for - * you if you reach the last page of results. - * - * @example - * ```ts - * const findOptions: SavedObjectsFindOptions = { - * type: 'visualization', - * search: 'foo*', - * perPage: 100, - * }; - * - * const finder = createPointInTimeFinder({ - * logger, - * savedObjectsClient, - * findOptions, - * }); - * - * const responses: SavedObjectFindResponse[] = []; - * for await (const response of finder.find()) { - * responses.push(...response); - * if (doneSearching) { - * await finder.close(); - * } - * } - * ``` + * @public */ -export function createPointInTimeFinder({ - findOptions, - logger, - savedObjectsClient, -}: { - findOptions: SavedObjectsFindOptions; +export interface SavedObjectsCreatePointInTimeFinderDependencies { + client: Pick; +} + +/** + * @internal + */ +export interface PointInTimeFinderDependencies + extends SavedObjectsCreatePointInTimeFinderDependencies { logger: Logger; - savedObjectsClient: SavedObjectsClientContract; -}) { - return new PointInTimeFinder({ findOptions, logger, savedObjectsClient }); +} + +/** @public */ +export interface ISavedObjectsPointInTimeFinder { + /** + * An async generator which wraps calls to `savedObjectsClient.find` and + * iterates over multiple pages of results using `_pit` and `search_after`. + * This will open a new Point-In-Time (PIT), and continue paging until a set + * of results is received that's smaller than the designated `perPage` size. + */ + find: () => AsyncGenerator; + /** + * Closes the Point-In-Time associated with this finder instance. + * + * Once you have retrieved all of the results you need, it is recommended + * to call `close()` to clean up the PIT and prevent Elasticsearch from + * consuming resources unnecessarily. This is only required if you are + * done iterating and have not yet paged through all of the results: the + * PIT will automatically be closed for you once you reach the last page + * of results, or if the underlying call to `find` fails for any reason. + */ + close: () => Promise; } /** * @internal */ -export class PointInTimeFinder { +export class PointInTimeFinder implements ISavedObjectsPointInTimeFinder { readonly #log: Logger; - readonly #savedObjectsClient: SavedObjectsClientContract; + readonly #client: PointInTimeFinderClient; readonly #findOptions: SavedObjectsFindOptions; #open: boolean = false; #pitId?: string; - constructor({ - findOptions, - logger, - savedObjectsClient, - }: { - findOptions: SavedObjectsFindOptions; - logger: Logger; - savedObjectsClient: SavedObjectsClientContract; - }) { - this.#log = logger; - this.#savedObjectsClient = savedObjectsClient; + constructor( + findOptions: SavedObjectsCreatePointInTimeFinderOptions, + { logger, client }: PointInTimeFinderDependencies + ) { + this.#log = logger.get('point-in-time-finder'); + this.#client = client; this.#findOptions = { // Default to 1000 items per page as a tradeoff between // speed and memory consumption. @@ -110,7 +107,7 @@ export class PointInTimeFinder { lastResultsCount = results.saved_objects.length; lastHitSortValue = this.getLastHitSortValue(results); - this.#log.debug(`Collected [${lastResultsCount}] saved objects for export.`); + this.#log.debug(`Collected [${lastResultsCount}] saved objects`); // Close PIT if this was our last page if (this.#pitId && lastResultsCount < this.#findOptions.perPage!) { @@ -129,7 +126,7 @@ export class PointInTimeFinder { try { if (this.#pitId) { this.#log.debug(`Closing PIT for types [${this.#findOptions.type}]`); - await this.#savedObjectsClient.closePointInTime(this.#pitId); + await this.#client.closePointInTime(this.#pitId); this.#pitId = undefined; } this.#open = false; @@ -141,13 +138,14 @@ export class PointInTimeFinder { private async open() { try { - const { id } = await this.#savedObjectsClient.openPointInTimeForType(this.#findOptions.type); + const { id } = await this.#client.openPointInTimeForType(this.#findOptions.type); this.#pitId = id; this.#open = true; } catch (e) { - // Since `find` swallows 404s, it is expected that exporter will do the same, + // Since `find` swallows 404s, it is expected that finder will do the same, // so we only rethrow non-404 errors here. - if (e.output.statusCode !== 404) { + if (e.output?.statusCode !== 404) { + this.#log.error(`Failed to open PIT for types [${this.#findOptions.type}]`); throw e; } this.#log.debug(`Unable to open PIT for types [${this.#findOptions.type}]: 404 ${e}`); @@ -164,7 +162,7 @@ export class PointInTimeFinder { searchAfter?: estypes.Id[]; }) { try { - return await this.#savedObjectsClient.find({ + return await this.#client.find({ // Sort fields are required to use searchAfter, so we set some defaults here sortField: 'updated_at', sortOrder: 'desc', diff --git a/src/core/server/saved_objects/service/lib/repository.mock.ts b/src/core/server/saved_objects/service/lib/repository.mock.ts index a3610b1e437e2..a2092e0571808 100644 --- a/src/core/server/saved_objects/service/lib/repository.mock.ts +++ b/src/core/server/saved_objects/service/lib/repository.mock.ts @@ -6,26 +6,36 @@ * Side Public License, v 1. */ +import { savedObjectsPointInTimeFinderMock } from './point_in_time_finder.mock'; import { ISavedObjectsRepository } from './repository'; -const create = (): jest.Mocked => ({ - checkConflicts: jest.fn(), - create: jest.fn(), - bulkCreate: jest.fn(), - bulkUpdate: jest.fn(), - delete: jest.fn(), - bulkGet: jest.fn(), - find: jest.fn(), - get: jest.fn(), - closePointInTime: jest.fn(), - openPointInTimeForType: jest.fn().mockResolvedValue({ id: 'some_pit_id' }), - resolve: jest.fn(), - update: jest.fn(), - addToNamespaces: jest.fn(), - deleteFromNamespaces: jest.fn(), - deleteByNamespace: jest.fn(), - incrementCounter: jest.fn(), - removeReferencesTo: jest.fn(), -}); +const create = () => { + const mock: jest.Mocked = { + checkConflicts: jest.fn(), + create: jest.fn(), + bulkCreate: jest.fn(), + bulkUpdate: jest.fn(), + delete: jest.fn(), + bulkGet: jest.fn(), + find: jest.fn(), + get: jest.fn(), + closePointInTime: jest.fn(), + createPointInTimeFinder: jest.fn(), + openPointInTimeForType: jest.fn().mockResolvedValue({ id: 'some_pit_id' }), + resolve: jest.fn(), + update: jest.fn(), + addToNamespaces: jest.fn(), + deleteFromNamespaces: jest.fn(), + deleteByNamespace: jest.fn(), + incrementCounter: jest.fn(), + removeReferencesTo: jest.fn(), + }; + + mock.createPointInTimeFinder = savedObjectsPointInTimeFinderMock.create({ + savedObjectsMock: mock, + }); + + return mock; +}; export const savedObjectsRepositoryMock = { create }; diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index 83ea80567d261..37572c83e4c88 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -6,10 +6,14 @@ * Side Public License, v 1. */ +import { pointInTimeFinderMock } from './repository.test.mock'; + import { SavedObjectsRepository } from './repository'; import * as getSearchDslNS from './search_dsl/search_dsl'; import { SavedObjectsErrorHelpers } from './errors'; +import { PointInTimeFinder } from './point_in_time_finder'; import { ALL_NAMESPACES_STRING } from './utils'; +import { loggerMock } from '../../../logging/logger.mock'; import { SavedObjectsSerializer } from '../../serialization'; import { encodeHitVersion } from '../../version'; import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; @@ -39,6 +43,7 @@ describe('SavedObjectsRepository', () => { let client; let savedObjectsRepository; let migrator; + let logger; let serializer; const mockTimestamp = '2017-08-14T15:49:14.886Z'; @@ -238,11 +243,13 @@ describe('SavedObjectsRepository', () => { }; beforeEach(() => { + pointInTimeFinderMock.mockClear(); client = elasticsearchClientMock.createElasticsearchClient(); migrator = mockKibanaMigrator.create(); documentMigrator.prepareMigrations(); migrator.migrateDocument = jest.fn().mockImplementation(documentMigrator.migrate); migrator.runMigrations = async () => ({ status: 'skipped' }); + logger = loggerMock.create(); // create a mock serializer "shim" so we can track function calls, but use the real serializer's implementation serializer = { @@ -269,6 +276,7 @@ describe('SavedObjectsRepository', () => { typeRegistry: registry, serializer, allowedTypes, + logger, }); savedObjectsRepository._getCurrentTime = jest.fn(() => mockTimestamp); @@ -4635,4 +4643,31 @@ describe('SavedObjectsRepository', () => { }); }); }); + + describe('#createPointInTimeFinder', () => { + it('returns a new PointInTimeFinder instance', async () => { + const result = await savedObjectsRepository.createPointInTimeFinder({}, {}); + expect(result).toBeInstanceOf(PointInTimeFinder); + }); + + it('calls PointInTimeFinder with the provided options and dependencies', async () => { + const options = Symbol(); + const dependencies = { + client: { + find: Symbol(), + openPointInTimeForType: Symbol(), + closePointInTime: Symbol(), + }, + }; + + await savedObjectsRepository.createPointInTimeFinder(options, dependencies); + expect(pointInTimeFinderMock).toHaveBeenCalledWith( + options, + expect.objectContaining({ + ...dependencies, + logger, + }) + ); + }); + }); }); diff --git a/src/core/server/saved_objects/service/lib/repository.test.mock.ts b/src/core/server/saved_objects/service/lib/repository.test.mock.ts new file mode 100644 index 0000000000000..3eba77b465819 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/repository.test.mock.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const pointInTimeFinderMock = jest.fn(); +jest.doMock('./point_in_time_finder', () => ({ + PointInTimeFinder: pointInTimeFinderMock, +})); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 37d03f2ef4f91..aa1e62c1652ca 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -8,8 +8,15 @@ import { omit, isObject } from 'lodash'; import type { estypes } from '@elastic/elasticsearch'; -import { ElasticsearchClient, DeleteDocumentResponse } from '../../../elasticsearch'; +import type { ElasticsearchClient } from '../../../elasticsearch/'; +import type { Logger } from '../../../logging'; import { getRootPropertiesObjects, IndexMapping } from '../../mappings'; +import { + ISavedObjectsPointInTimeFinder, + PointInTimeFinder, + SavedObjectsCreatePointInTimeFinderOptions, + SavedObjectsCreatePointInTimeFinderDependencies, +} from './point_in_time_finder'; import { createRepositoryEsClient, RepositoryEsClient } from './repository_es_client'; import { getSearchDsl } from './search_dsl'; import { includedFields } from './included_fields'; @@ -85,6 +92,7 @@ export interface SavedObjectsRepositoryOptions { serializer: SavedObjectsSerializer; migrator: IKibanaMigrator; allowedTypes: string[]; + logger: Logger; } /** @@ -144,6 +152,7 @@ export class SavedObjectsRepository { private _allowedTypes: string[]; private readonly client: RepositoryEsClient; private _serializer: SavedObjectsSerializer; + private _logger: Logger; /** * A factory function for creating SavedObjectRepository instances. @@ -158,6 +167,7 @@ export class SavedObjectsRepository { typeRegistry: SavedObjectTypeRegistry, indexName: string, client: ElasticsearchClient, + logger: Logger, includedHiddenTypes: string[] = [], injectedConstructor: any = SavedObjectsRepository ): ISavedObjectsRepository { @@ -183,6 +193,7 @@ export class SavedObjectsRepository { serializer, allowedTypes, client, + logger, }); } @@ -195,6 +206,7 @@ export class SavedObjectsRepository { serializer, migrator, allowedTypes = [], + logger, } = options; // It's important that we migrate documents / mark them as up-to-date @@ -214,6 +226,7 @@ export class SavedObjectsRepository { } this._allowedTypes = allowedTypes; this._serializer = serializer; + this._logger = logger; } /** @@ -624,7 +637,7 @@ export class SavedObjectsRepository { } } - const { body, statusCode } = await this.client.delete( + const { body, statusCode } = await this.client.delete( { id: rawId, index: this.getIndexForType(type), @@ -1330,7 +1343,7 @@ export class SavedObjectsRepository { return { namespaces: doc.namespaces }; } else { // if there are no namespaces remaining, delete the saved object - const { body, statusCode } = await this.client.delete( + const { body, statusCode } = await this.client.delete( { id: this._serializer.generateRawId(undefined, type, id), refresh, @@ -1810,6 +1823,9 @@ export class SavedObjectsRepository { * Opens a Point In Time (PIT) against the indices for the specified Saved Object types. * The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. * + * Only use this API if you have an advanced use case that's not solved by the + * {@link SavedObjectsRepository.createPointInTimeFinder} method. + * * @example * ```ts * const { id } = await savedObjectsClient.openPointInTimeForType( @@ -1879,6 +1895,9 @@ export class SavedObjectsRepository { * via the Elasticsearch client, and is included in the Saved Objects Client * as a convenience for consumers who are using `openPointInTimeForType`. * + * Only use this API if you have an advanced use case that's not solved by the + * {@link SavedObjectsRepository.createPointInTimeFinder} method. + * * @remarks * While the `keepAlive` that is provided will cause a PIT to automatically close, * it is highly recommended to explicitly close a PIT when you are done with it @@ -1923,6 +1942,62 @@ export class SavedObjectsRepository { return body; } + /** + * Returns a {@link ISavedObjectsPointInTimeFinder} to help page through + * large sets of saved objects. We strongly recommend using this API for + * any `find` queries that might return more than 1000 saved objects, + * however this API is only intended for use in server-side "batch" + * processing of objects where you are collecting all objects in memory + * or streaming them back to the client. + * + * Do NOT use this API in a route handler to facilitate paging through + * saved objects on the client-side unless you are streaming all of the + * results back to the client at once. Because the returned generator is + * stateful, you cannot rely on subsequent http requests retrieving new + * pages from the same Kibana server in multi-instance deployments. + * + * This generator wraps calls to {@link SavedObjectsRepository.find} and + * iterates over multiple pages of results using `_pit` and `search_after`. + * This will open a new Point-In-Time (PIT), and continue paging until a + * set of results is received that's smaller than the designated `perPage`. + * + * Once you have retrieved all of the results you need, it is recommended + * to call `close()` to clean up the PIT and prevent Elasticsearch from + * consuming resources unnecessarily. This is only required if you are + * done iterating and have not yet paged through all of the results: the + * PIT will automatically be closed for you once you reach the last page + * of results, or if the underlying call to `find` fails for any reason. + * + * @example + * ```ts + * const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { + * type: 'visualization', + * search: 'foo*', + * perPage: 100, + * }; + * + * const finder = savedObjectsClient.createPointInTimeFinder(findOptions); + * + * const responses: SavedObjectFindResponse[] = []; + * for await (const response of finder.find()) { + * responses.push(...response); + * if (doneSearching) { + * await finder.close(); + * } + * } + * ``` + */ + createPointInTimeFinder( + findOptions: SavedObjectsCreatePointInTimeFinderOptions, + dependencies?: SavedObjectsCreatePointInTimeFinderDependencies + ): ISavedObjectsPointInTimeFinder { + return new PointInTimeFinder(findOptions, { + logger: this._logger, + client: this, + ...dependencies, + }); + } + /** * Returns index specified by the given type or the default index * diff --git a/src/core/server/saved_objects/service/lib/repository_create_repository.test.ts b/src/core/server/saved_objects/service/lib/repository_create_repository.test.ts index 26aa152c630ad..9d9a2eb14b495 100644 --- a/src/core/server/saved_objects/service/lib/repository_create_repository.test.ts +++ b/src/core/server/saved_objects/service/lib/repository_create_repository.test.ts @@ -10,12 +10,14 @@ import { SavedObjectsRepository } from './repository'; import { mockKibanaMigrator } from '../../migrations/kibana/kibana_migrator.mock'; import { KibanaMigrator } from '../../migrations'; import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; +import { loggerMock, MockedLogger } from '../../../logging/logger.mock'; jest.mock('./repository'); const { SavedObjectsRepository: originalRepository } = jest.requireActual('./repository'); describe('SavedObjectsRepository#createRepository', () => { + let logger: MockedLogger; const callAdminCluster = jest.fn(); const typeRegistry = new SavedObjectTypeRegistry(); @@ -59,6 +61,7 @@ describe('SavedObjectsRepository#createRepository', () => { const RepositoryConstructor = (SavedObjectsRepository as unknown) as jest.Mock; beforeEach(() => { + logger = loggerMock.create(); RepositoryConstructor.mockClear(); }); @@ -69,6 +72,7 @@ describe('SavedObjectsRepository#createRepository', () => { typeRegistry, '.kibana-test', callAdminCluster, + logger, ['unMappedType1', 'unmappedType2'] ); } catch (e) { @@ -84,6 +88,7 @@ describe('SavedObjectsRepository#createRepository', () => { typeRegistry, '.kibana-test', callAdminCluster, + logger, [], SavedObjectsRepository ); @@ -102,6 +107,7 @@ describe('SavedObjectsRepository#createRepository', () => { typeRegistry, '.kibana-test', callAdminCluster, + logger, ['hiddenType', 'hiddenType', 'hiddenType'], SavedObjectsRepository ); diff --git a/src/core/server/saved_objects/service/saved_objects_client.mock.ts b/src/core/server/saved_objects/service/saved_objects_client.mock.ts index ecca652cace37..544e92e32f1a1 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.mock.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.mock.ts @@ -8,9 +8,10 @@ import { SavedObjectsClientContract } from '../types'; import { SavedObjectsErrorHelpers } from './lib/errors'; +import { savedObjectsPointInTimeFinderMock } from './lib/point_in_time_finder.mock'; -const create = () => - (({ +const create = () => { + const mock = ({ errors: SavedObjectsErrorHelpers, create: jest.fn(), bulkCreate: jest.fn(), @@ -21,12 +22,20 @@ const create = () => find: jest.fn(), get: jest.fn(), closePointInTime: jest.fn(), + createPointInTimeFinder: jest.fn(), openPointInTimeForType: jest.fn().mockResolvedValue({ id: 'some_pit_id' }), resolve: jest.fn(), update: jest.fn(), addToNamespaces: jest.fn(), deleteFromNamespaces: jest.fn(), removeReferencesTo: jest.fn(), - } as unknown) as jest.Mocked); + } as unknown) as jest.Mocked; + + mock.createPointInTimeFinder = savedObjectsPointInTimeFinderMock.create({ + savedObjectsMock: mock, + }); + + return mock; +}; export const savedObjectsClientMock = { create }; diff --git a/src/core/server/saved_objects/service/saved_objects_client.test.js b/src/core/server/saved_objects/service/saved_objects_client.test.js index 7cbddaf195dc9..29381c7e418b5 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.test.js +++ b/src/core/server/saved_objects/service/saved_objects_client.test.js @@ -54,6 +54,45 @@ test(`#bulkCreate`, async () => { expect(result).toBe(returnValue); }); +describe(`#createPointInTimeFinder`, () => { + test(`calls repository with options and default dependencies`, () => { + const returnValue = Symbol(); + const mockRepository = { + createPointInTimeFinder: jest.fn().mockReturnValue(returnValue), + }; + const client = new SavedObjectsClient(mockRepository); + + const options = Symbol(); + const result = client.createPointInTimeFinder(options); + + expect(mockRepository.createPointInTimeFinder).toHaveBeenCalledWith(options, { + client, + }); + expect(result).toBe(returnValue); + }); + + test(`calls repository with options and custom dependencies`, () => { + const returnValue = Symbol(); + const mockRepository = { + createPointInTimeFinder: jest.fn().mockReturnValue(returnValue), + }; + const client = new SavedObjectsClient(mockRepository); + + const options = Symbol(); + const dependencies = { + client: { + find: Symbol(), + openPointInTimeForType: Symbol(), + closePointInTime: Symbol(), + }, + }; + const result = client.createPointInTimeFinder(options, dependencies); + + expect(mockRepository.createPointInTimeFinder).toHaveBeenCalledWith(options, dependencies); + expect(result).toBe(returnValue); + }); +}); + test(`#delete`, async () => { const returnValue = Symbol(); const mockRepository = { diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 2b314f643f792..9a0ccb88d3555 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -6,7 +6,12 @@ * Side Public License, v 1. */ -import { ISavedObjectsRepository } from './lib'; +import type { + ISavedObjectsRepository, + ISavedObjectsPointInTimeFinder, + SavedObjectsCreatePointInTimeFinderOptions, + SavedObjectsCreatePointInTimeFinderDependencies, +} from './lib'; import { SavedObject, SavedObjectError, @@ -587,6 +592,9 @@ export class SavedObjectsClient { * Opens a Point In Time (PIT) against the indices for the specified Saved Object types. * The returned `id` can then be passed to {@link SavedObjectsClient.find} to search * against that PIT. + * + * Only use this API if you have an advanced use case that's not solved by the + * {@link SavedObjectsClient.createPointInTimeFinder} method. */ async openPointInTimeForType( type: string | string[], @@ -599,8 +607,67 @@ export class SavedObjectsClient { * Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the * Elasticsearch client, and is included in the Saved Objects Client as a convenience * for consumers who are using {@link SavedObjectsClient.openPointInTimeForType}. + * + * Only use this API if you have an advanced use case that's not solved by the + * {@link SavedObjectsClient.createPointInTimeFinder} method. */ async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { return await this._repository.closePointInTime(id, options); } + + /** + * Returns a {@link ISavedObjectsPointInTimeFinder} to help page through + * large sets of saved objects. We strongly recommend using this API for + * any `find` queries that might return more than 1000 saved objects, + * however this API is only intended for use in server-side "batch" + * processing of objects where you are collecting all objects in memory + * or streaming them back to the client. + * + * Do NOT use this API in a route handler to facilitate paging through + * saved objects on the client-side unless you are streaming all of the + * results back to the client at once. Because the returned generator is + * stateful, you cannot rely on subsequent http requests retrieving new + * pages from the same Kibana server in multi-instance deployments. + * + * The generator wraps calls to {@link SavedObjectsClient.find} and iterates + * over multiple pages of results using `_pit` and `search_after`. This will + * open a new Point-In-Time (PIT), and continue paging until a set of + * results is received that's smaller than the designated `perPage`. + * + * Once you have retrieved all of the results you need, it is recommended + * to call `close()` to clean up the PIT and prevent Elasticsearch from + * consuming resources unnecessarily. This is only required if you are + * done iterating and have not yet paged through all of the results: the + * PIT will automatically be closed for you once you reach the last page + * of results, or if the underlying call to `find` fails for any reason. + * + * @example + * ```ts + * const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { + * type: 'visualization', + * search: 'foo*', + * perPage: 100, + * }; + * + * const finder = savedObjectsClient.createPointInTimeFinder(findOptions); + * + * const responses: SavedObjectFindResponse[] = []; + * for await (const response of finder.find()) { + * responses.push(...response); + * if (doneSearching) { + * await finder.close(); + * } + * } + * ``` + */ + createPointInTimeFinder( + findOptions: SavedObjectsCreatePointInTimeFinderOptions, + dependencies?: SavedObjectsCreatePointInTimeFinderDependencies + ): ISavedObjectsPointInTimeFinder { + return this._repository.createPointInTimeFinder(findOptions, { + client: this, + // Include dependencies last so that SO client wrappers have their settings applied. + ...dependencies, + }); + } } diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index dda3180d20093..73f8a44075162 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1178,6 +1178,12 @@ export type ISavedObjectsExporter = PublicMethodsOf; // @public (undocumented) export type ISavedObjectsImporter = PublicMethodsOf; +// @public (undocumented) +export interface ISavedObjectsPointInTimeFinder { + close: () => Promise; + find: () => AsyncGenerator; +} + // @public export type ISavedObjectsRepository = Pick; @@ -2220,6 +2226,7 @@ export class SavedObjectsClient { checkConflicts(objects?: SavedObjectsCheckConflictsObject[], options?: SavedObjectsBaseOptions): Promise; closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions): Promise; create(type: string, attributes: T, options?: SavedObjectsCreateOptions): Promise>; + createPointInTimeFinder(findOptions: SavedObjectsCreatePointInTimeFinderOptions, dependencies?: SavedObjectsCreatePointInTimeFinderDependencies): ISavedObjectsPointInTimeFinder; delete(type: string, id: string, options?: SavedObjectsDeleteOptions): Promise<{}>; deleteFromNamespaces(type: string, id: string, namespaces: string[], options?: SavedObjectsDeleteFromNamespacesOptions): Promise; // (undocumented) @@ -2322,6 +2329,15 @@ export interface SavedObjectsCreateOptions extends SavedObjectsBaseOptions { version?: string; } +// @public (undocumented) +export interface SavedObjectsCreatePointInTimeFinderDependencies { + // (undocumented) + client: Pick; +} + +// @public (undocumented) +export type SavedObjectsCreatePointInTimeFinderOptions = Omit; + // @public (undocumented) export interface SavedObjectsDeleteByNamespaceOptions extends SavedObjectsBaseOptions { refresh?: boolean; @@ -2812,10 +2828,11 @@ export class SavedObjectsRepository { checkConflicts(objects?: SavedObjectsCheckConflictsObject[], options?: SavedObjectsBaseOptions): Promise; closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions): Promise; create(type: string, attributes: T, options?: SavedObjectsCreateOptions): Promise>; + createPointInTimeFinder(findOptions: SavedObjectsCreatePointInTimeFinderOptions, dependencies?: SavedObjectsCreatePointInTimeFinderDependencies): ISavedObjectsPointInTimeFinder; // Warning: (ae-forgotten-export) The symbol "IKibanaMigrator" needs to be exported by the entry point index.d.ts // // @internal - static createRepository(migrator: IKibanaMigrator, typeRegistry: SavedObjectTypeRegistry, indexName: string, client: ElasticsearchClient, includedHiddenTypes?: string[], injectedConstructor?: any): ISavedObjectsRepository; + static createRepository(migrator: IKibanaMigrator, typeRegistry: SavedObjectTypeRegistry, indexName: string, client: ElasticsearchClient, logger: Logger, includedHiddenTypes?: string[], injectedConstructor?: any): ISavedObjectsRepository; delete(type: string, id: string, options?: SavedObjectsDeleteOptions): Promise<{}>; deleteByNamespace(namespace: string, options?: SavedObjectsDeleteByNamespaceOptions): Promise; deleteFromNamespaces(type: string, id: string, namespaces: string[], options?: SavedObjectsDeleteFromNamespacesOptions): Promise; diff --git a/src/dev/cli_dev_mode/cli_dev_mode.test.ts b/src/dev/cli_dev_mode/cli_dev_mode.test.ts index 54c49ce21505f..9ace543a8929b 100644 --- a/src/dev/cli_dev_mode/cli_dev_mode.test.ts +++ b/src/dev/cli_dev_mode/cli_dev_mode.test.ts @@ -31,6 +31,9 @@ const { Optimizer } = jest.requireMock('./optimizer'); jest.mock('./dev_server'); const { DevServer } = jest.requireMock('./dev_server'); +jest.mock('@kbn/dev-utils/target/ci_stats_reporter'); +const { CiStatsReporter } = jest.requireMock('@kbn/dev-utils/target/ci_stats_reporter'); + jest.mock('./get_server_watch_paths', () => ({ getServerWatchPaths: jest.fn(() => ({ watchPaths: [''], @@ -208,6 +211,11 @@ describe('#start()/#stop()', () => { run$: devServerRun$, }; }); + CiStatsReporter.fromEnv.mockImplementation(() => { + return { + isEnabled: jest.fn().mockReturnValue(false), + }; + }); }); afterEach(() => { diff --git a/src/dev/cli_dev_mode/cli_dev_mode.ts b/src/dev/cli_dev_mode/cli_dev_mode.ts index 1eed8b14aed4a..f4f95f20daeef 100644 --- a/src/dev/cli_dev_mode/cli_dev_mode.ts +++ b/src/dev/cli_dev_mode/cli_dev_mode.ts @@ -10,7 +10,16 @@ import Path from 'path'; import { REPO_ROOT, CiStatsReporter } from '@kbn/dev-utils'; import * as Rx from 'rxjs'; -import { map, mapTo, filter, take, tap, distinctUntilChanged, switchMap } from 'rxjs/operators'; +import { + map, + mapTo, + filter, + take, + tap, + distinctUntilChanged, + switchMap, + concatMap, +} from 'rxjs/operators'; import { CliArgs } from '../../core/server/config'; import { LegacyConfig } from '../../core/server/legacy'; @@ -167,29 +176,10 @@ export class CliDevMode { this.subscription = new Rx.Subscription(); this.startTime = Date.now(); - this.subscription.add( - this.getStarted$() - .pipe( - switchMap(async (success) => { - const reporter = CiStatsReporter.fromEnv(this.log.toolingLog); - await reporter.timings({ - timings: [ - { - group: 'yarn start', - id: 'started', - ms: Date.now() - this.startTime!, - meta: { success }, - }, - ], - }); - }) - ) - .subscribe({ - error: (error) => { - this.log.bad(`[ci-stats/timings] unable to record startup time:`, error.stack); - }, - }) - ); + const reporter = CiStatsReporter.fromEnv(this.log.toolingLog); + if (reporter.isEnabled()) { + this.subscription.add(this.reportTimings(reporter)); + } if (basePathProxy) { const serverReady$ = new Rx.BehaviorSubject(false); @@ -245,6 +235,64 @@ export class CliDevMode { this.subscription.add(this.devServer.run$.subscribe(this.observer('dev server'))); } + private reportTimings(reporter: CiStatsReporter) { + const sub = new Rx.Subscription(); + + sub.add( + this.getStarted$() + .pipe( + concatMap(async (success) => { + await reporter.timings({ + timings: [ + { + group: 'yarn start', + id: 'started', + ms: Date.now() - this.startTime!, + meta: { success }, + }, + ], + }); + }) + ) + .subscribe({ + error: (error) => { + this.log.bad(`[ci-stats/timings] unable to record startup time:`, error.stack); + }, + }) + ); + + sub.add( + this.devServer + .getRestartTime$() + .pipe( + concatMap(async ({ ms }, i) => { + await reporter.timings({ + timings: [ + { + group: 'yarn start', + id: 'dev server restart', + ms, + meta: { + sequence: i + 1, + }, + }, + ], + }); + }) + ) + .subscribe({ + error: (error) => { + this.log.bad( + `[ci-stats/timings] unable to record dev server restart time:`, + error.stack + ); + }, + }) + ); + + return sub; + } + /** * returns an observable that emits once the dev server and optimizer are started, emits * true if they both started successfully, otherwise false diff --git a/src/dev/cli_dev_mode/dev_server.test.ts b/src/dev/cli_dev_mode/dev_server.test.ts index c296c7caca63a..9962a9a285a42 100644 --- a/src/dev/cli_dev_mode/dev_server.test.ts +++ b/src/dev/cli_dev_mode/dev_server.test.ts @@ -15,6 +15,8 @@ import { extendedEnvSerializer } from './test_helpers'; import { DevServer, Options } from './dev_server'; import { TestLog } from './log'; +jest.useFakeTimers('modern'); + class MockProc extends EventEmitter { public readonly signalsSent: string[] = []; @@ -91,6 +93,17 @@ const run = (server: DevServer) => { return subscription; }; +const collect = (stream: Rx.Observable) => { + const events: T[] = []; + const subscription = stream.subscribe({ + next(item) { + events.push(item); + }, + }); + subscriptions.push(subscription); + return events; +}; + afterEach(() => { if (currentProc) { currentProc.removeAllListeners(); @@ -107,6 +120,9 @@ describe('#run$', () => { it('starts the dev server with the right options', () => { run(new DevServer(defaultOptions)).unsubscribe(); + // ensure that FORCE_COLOR is in the env for consistency in snapshot + process.env.FORCE_COLOR = process.env.FORCE_COLOR || 'true'; + expect(execa.node.mock.calls).toMatchInlineSnapshot(` Array [ Array [ @@ -305,7 +321,106 @@ describe('#run$', () => { expect(currentProc.signalsSent).toEqual([]); sigint$.next(); expect(currentProc.signalsSent).toEqual(['SIGINT']); - await new Promise((resolve) => setTimeout(resolve, 1000)); + jest.advanceTimersByTime(100); expect(currentProc.signalsSent).toEqual(['SIGINT', 'SIGKILL']); }); }); + +describe('#getPhase$', () => { + it('emits "starting" when run$ is subscribed then emits "fatal exit" when server exits with code > 0, then starting once watcher fires and "listening" when the server is ready', () => { + const server = new DevServer(defaultOptions); + const events = collect(server.getPhase$()); + + expect(events).toEqual([]); + run(server); + expect(events).toEqual(['starting']); + events.length = 0; + + isProc(currentProc); + currentProc.mockExit(2); + expect(events).toEqual(['fatal exit']); + events.length = 0; + + restart$.next(); + expect(events).toEqual(['starting']); + events.length = 0; + + currentProc.mockListening(); + expect(events).toEqual(['listening']); + }); +}); + +describe('#getRestartTime$()', () => { + it('does not send event if server does not start listening before starting again', () => { + const server = new DevServer(defaultOptions); + const phases = collect(server.getPhase$()); + const events = collect(server.getRestartTime$()); + run(server); + + isProc(currentProc); + restart$.next(); + jest.advanceTimersByTime(1000); + restart$.next(); + jest.advanceTimersByTime(1000); + restart$.next(); + expect(phases).toMatchInlineSnapshot(` + Array [ + "starting", + "starting", + "starting", + "starting", + ] + `); + expect(events).toEqual([]); + }); + + it('reports restart times', () => { + const server = new DevServer(defaultOptions); + const phases = collect(server.getPhase$()); + const events = collect(server.getRestartTime$()); + + run(server); + isProc(currentProc); + + restart$.next(); + currentProc.mockExit(1); + restart$.next(); + restart$.next(); + restart$.next(); + currentProc.mockExit(1); + restart$.next(); + jest.advanceTimersByTime(1234); + currentProc.mockListening(); + restart$.next(); + restart$.next(); + jest.advanceTimersByTime(5678); + currentProc.mockListening(); + + expect(phases).toMatchInlineSnapshot(` + Array [ + "starting", + "starting", + "fatal exit", + "starting", + "starting", + "starting", + "fatal exit", + "starting", + "listening", + "starting", + "starting", + "listening", + ] + `); + expect(events).toMatchInlineSnapshot(` + Array [ + Object { + "ms": 1234, + }, + Object { + "ms": 5678, + }, + ] + `); + }); +}); diff --git a/src/dev/cli_dev_mode/dev_server.ts b/src/dev/cli_dev_mode/dev_server.ts index a4e32a40665e3..3daf298c82324 100644 --- a/src/dev/cli_dev_mode/dev_server.ts +++ b/src/dev/cli_dev_mode/dev_server.ts @@ -16,6 +16,7 @@ import { share, mergeMap, switchMap, + scan, takeUntil, ignoreElements, } from 'rxjs/operators'; @@ -73,6 +74,32 @@ export class DevServer { return this.phase$.asObservable(); } + /** + * returns an observable of objects describing server start time. + */ + getRestartTime$() { + return this.phase$.pipe( + scan((acc: undefined | { phase: string; time: number }, phase) => { + if (phase === 'starting') { + return { phase, time: Date.now() }; + } + + if (phase === 'listening' && acc?.phase === 'starting') { + return { phase, time: Date.now() - acc.time }; + } + + return undefined; + }, undefined), + mergeMap((desc) => { + if (desc?.phase !== 'listening') { + return []; + } + + return [{ ms: desc.time }]; + }) + ); + } + /** * Run the Kibana server * diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index c23736883e0e6..12458d7a74d9f 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -1226,7 +1226,7 @@ export class Plugin implements Plugin_2 Promise; }; indexPatterns: { - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: import("../../../core/server").ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: import("../../../core/server").ElasticsearchClient) => Promise; }; search: ISearchStart>; }; diff --git a/src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts b/src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts index b53e5328f21ba..ad84518af9de3 100644 --- a/src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts +++ b/src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts @@ -22,6 +22,8 @@ const fields = [ type: 'date', scripted: false, filterable: true, + aggregatable: true, + sortable: true, }, { name: 'message', @@ -34,12 +36,14 @@ const fields = [ type: 'string', scripted: false, filterable: true, + aggregatable: true, }, { name: 'bytes', type: 'number', scripted: false, filterable: true, + aggregatable: true, }, { name: 'scripted', @@ -55,14 +59,14 @@ fields.getByName = (name: string) => { const indexPattern = ({ id: 'index-pattern-with-timefield-id', - title: 'index-pattern-without-timefield', + title: 'index-pattern-with-timefield', metaFields: ['_index', '_score'], flattenHit: undefined, formatHit: jest.fn((hit) => hit._source), fields, getComputedFields: () => ({}), getSourceFiltering: () => ({}), - getFieldByName: () => ({}), + getFieldByName: (name: string) => fields.getByName(name), timeFieldName: 'timestamp', } as unknown) as IndexPattern; diff --git a/src/plugins/discover/public/__mocks__/ui_settings.ts b/src/plugins/discover/public/__mocks__/ui_settings.ts index 8bc6de1b9ca41..e021a39a568e9 100644 --- a/src/plugins/discover/public/__mocks__/ui_settings.ts +++ b/src/plugins/discover/public/__mocks__/ui_settings.ts @@ -7,12 +7,14 @@ */ import { IUiSettingsClient } from 'kibana/public'; -import { SAMPLE_SIZE_SETTING } from '../../common'; +import { DEFAULT_COLUMNS_SETTING, SAMPLE_SIZE_SETTING } from '../../common'; export const uiSettingsMock = ({ get: (key: string) => { if (key === SAMPLE_SIZE_SETTING) { return 10; + } else if (key === DEFAULT_COLUMNS_SETTING) { + return ['default_column']; } }, } as unknown) as IUiSettingsClient; diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 4a761f2fefa65..2c80fc111c740 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -9,8 +9,6 @@ import _ from 'lodash'; import { merge, Subject, Subscription } from 'rxjs'; import { debounceTime } from 'rxjs/operators'; -import moment from 'moment'; -import dateMath from '@elastic/datemath'; import { i18n } from '@kbn/i18n'; import { createSearchSessionRestorationDataProvider, getState, splitState } from './discover_state'; import { RequestAdapter } from '../../../../inspector/public'; @@ -23,7 +21,6 @@ import { } from '../../../../data/public'; import { getSortArray } from './doc_table'; import indexTemplateLegacy from './discover_legacy.html'; -import { addHelpMenuToAppChrome } from '../components/help_menu/help_menu_util'; import { discoverResponseHandler } from './response_handler'; import { getAngularModule, @@ -36,25 +33,22 @@ import { subscribeWithScope, tabifyAggResponse, } from '../../kibana_services'; -import { - getRootBreadcrumbs, - getSavedSearchBreadcrumbs, - setBreadcrumbsTitle, -} from '../helpers/breadcrumbs'; +import { getRootBreadcrumbs, getSavedSearchBreadcrumbs } from '../helpers/breadcrumbs'; +import { getStateDefaults } from '../helpers/get_state_defaults'; +import { getResultState } from '../helpers/get_result_state'; import { validateTimeRange } from '../helpers/validate_time_range'; import { addFatalError } from '../../../../kibana_legacy/public'; import { - DEFAULT_COLUMNS_SETTING, SAMPLE_SIZE_SETTING, SEARCH_FIELDS_FROM_SOURCE, SEARCH_ON_PAGE_LOAD_SETTING, - SORT_DEFAULT_ORDER_SETTING, } from '../../../common'; import { loadIndexPattern, resolveIndexPattern } from '../helpers/resolve_index_pattern'; import { updateSearchSource } from '../helpers/update_search_source'; import { calcFieldCounts } from '../helpers/calc_field_counts'; -import { getDefaultSort } from './doc_table/lib/get_default_sort'; import { DiscoverSearchSessionManager } from './discover_search_session'; +import { applyAggsToSearchSource, getDimensions } from '../components/histogram'; +import { fetchStatuses } from '../components/constants'; const services = getServices(); @@ -70,13 +64,6 @@ const { uiSettings: config, } = getServices(); -const fetchStatuses = { - UNINITIALIZED: 'uninitialized', - LOADING: 'loading', - COMPLETE: 'complete', - ERROR: 'error', -}; - const app = getAngularModule(); app.config(($routeProvider) => { @@ -161,7 +148,7 @@ app.directive('discoverApp', function () { }; }); -function discoverController($route, $scope, Promise) { +function discoverController($route, $scope) { const { isDefault: isDefaultType } = indexPatternsUtils; const subscriptions = new Subscription(); const refetch$ = new Subject(); @@ -191,7 +178,14 @@ function discoverController($route, $scope, Promise) { }); const stateContainer = getState({ - getStateDefaults, + getStateDefaults: () => + getStateDefaults({ + config, + data, + indexPattern: $scope.indexPattern, + savedSearch, + searchSource: persistentSearchSource, + }), storeInSessionStorage: config.get('state:storeInSessionStorage'), history, toasts: core.notifications.toasts, @@ -232,6 +226,21 @@ function discoverController($route, $scope, Promise) { query: true, } ); + const showUnmappedFields = $scope.useNewFieldsApi; + const updateSearchSourceHelper = () => { + const { indexPattern, useNewFieldsApi } = $scope; + const { columns, sort } = $scope.state; + updateSearchSource({ + persistentSearchSource, + volatileSearchSource: $scope.volatileSearchSource, + indexPattern, + services, + sort, + columns, + useNewFieldsApi, + showUnmappedFields, + }); + }; const appStateUnsubscribe = appStateContainer.subscribe(async (newState) => { const { state: newStatePartial } = splitState(newState); @@ -293,21 +302,6 @@ function discoverController($route, $scope, Promise) { } ); - // update data source when filters update - subscriptions.add( - subscribeWithScope( - $scope, - filterManager.getUpdates$(), - { - next: () => { - $scope.state.filters = filterManager.getAppFilters(); - $scope.updateDataSource(); - }, - }, - (error) => addFatalError(core.fatalErrors, error) - ) - ); - $scope.opts = { // number of records to fetch, then paginate through sampleSize: config.get(SAMPLE_SIZE_SETTING), @@ -329,8 +323,19 @@ function discoverController($route, $scope, Promise) { requests: new RequestAdapter(), }); - $scope.minimumVisibleRows = 50; + const shouldSearchOnPageLoad = () => { + // A saved search is created on every page load, so we check the ID to see if we're loading a + // previously saved search or if it is just transient + return ( + config.get(SEARCH_ON_PAGE_LOAD_SETTING) || + savedSearch.id !== undefined || + timefilter.getRefreshInterval().pause === false || + searchSessionManager.hasSearchSessionIdInURL() + ); + }; + $scope.fetchStatus = fetchStatuses.UNINITIALIZED; + $scope.resultState = shouldSearchOnPageLoad() ? 'loading' : 'uninitialized'; let abortController; $scope.$on('$destroy', () => { @@ -385,157 +390,12 @@ function discoverController($route, $scope, Promise) { volatileSearchSource.setParent(persistentSearchSource); $scope.volatileSearchSource = volatileSearchSource; - - const pageTitleSuffix = savedSearch.id && savedSearch.title ? `: ${savedSearch.title}` : ''; - chrome.docTitle.change(`Discover${pageTitleSuffix}`); - - setBreadcrumbsTitle(savedSearch, chrome); - - function getDefaultColumns() { - if (savedSearch.columns.length > 0) { - return [...savedSearch.columns]; - } - return [...config.get(DEFAULT_COLUMNS_SETTING)]; - } - - function getStateDefaults() { - const query = - persistentSearchSource.getField('query') || data.query.queryString.getDefaultQuery(); - const sort = getSortArray(savedSearch.sort, $scope.indexPattern); - const columns = getDefaultColumns(); - - const defaultState = { - query, - sort: !sort.length - ? getDefaultSort($scope.indexPattern, config.get(SORT_DEFAULT_ORDER_SETTING, 'desc')) - : sort, - columns, - index: $scope.indexPattern.id, - interval: 'auto', - filters: _.cloneDeep(persistentSearchSource.getOwnField('filter')), - }; - if (savedSearch.grid) { - defaultState.grid = savedSearch.grid; - } - if (savedSearch.hideChart) { - defaultState.hideChart = savedSearch.hideChart; - } - - return defaultState; - } - $scope.state.index = $scope.indexPattern.id; $scope.state.sort = getSortArray($scope.state.sort, $scope.indexPattern); - const shouldSearchOnPageLoad = () => { - // A saved search is created on every page load, so we check the ID to see if we're loading a - // previously saved search or if it is just transient - return ( - config.get(SEARCH_ON_PAGE_LOAD_SETTING) || - savedSearch.id !== undefined || - timefilter.getRefreshInterval().pause === false || - searchSessionManager.hasSearchSessionIdInURL() - ); - }; - - const init = _.once(() => { - $scope.updateDataSource().then(async () => { - const fetch$ = merge( - refetch$, - filterManager.getFetches$(), - timefilter.getFetch$(), - timefilter.getAutoRefreshFetch$(), - data.query.queryString.getUpdates$(), - searchSessionManager.newSearchSessionIdFromURL$ - ).pipe(debounceTime(100)); - - subscriptions.add( - subscribeWithScope( - $scope, - fetch$, - { - next: $scope.fetch, - }, - (error) => addFatalError(core.fatalErrors, error) - ) - ); - subscriptions.add( - subscribeWithScope( - $scope, - timefilter.getTimeUpdate$(), - { - next: () => { - $scope.updateTime(); - }, - }, - (error) => addFatalError(core.fatalErrors, error) - ) - ); - - $scope.$watchMulti( - ['rows', 'fetchStatus'], - (function updateResultState() { - let prev = {}; - const status = { - UNINITIALIZED: 'uninitialized', - LOADING: 'loading', // initial data load - READY: 'ready', // results came back - NO_RESULTS: 'none', // no results came back - }; - - function pick(rows, oldRows, fetchStatus) { - // initial state, pretend we're already loading if we're about to execute a search so - // that the uninitilized message doesn't flash on screen - if (!$scope.fetchError && rows == null && oldRows == null && shouldSearchOnPageLoad()) { - return status.LOADING; - } - - if (fetchStatus === fetchStatuses.UNINITIALIZED) { - return status.UNINITIALIZED; - } - - const rowsEmpty = _.isEmpty(rows); - if (rowsEmpty && fetchStatus === fetchStatuses.LOADING) return status.LOADING; - else if (!rowsEmpty) return status.READY; - else return status.NO_RESULTS; - } - - return function () { - const current = { - rows: $scope.rows, - fetchStatus: $scope.fetchStatus, - }; - - $scope.resultState = pick( - current.rows, - prev.rows, - current.fetchStatus, - prev.fetchStatus - ); - - prev = current; - }; - })() - ); - - if (getTimeField()) { - setupVisualization(); - $scope.updateTime(); - } - - init.complete = true; - if (shouldSearchOnPageLoad()) { - refetch$.next(); - } - }); - }); - $scope.opts.fetch = $scope.fetch = function () { - // ignore requests to fetch before the app inits - if (!init.complete) return; $scope.fetchCounter++; $scope.fetchError = undefined; - $scope.minimumVisibleRows = 50; if (!validateTimeRange(timefilter.getTime(), toastNotifications)) { $scope.resultState = 'none'; return; @@ -546,17 +406,23 @@ function discoverController($route, $scope, Promise) { abortController = new AbortController(); const searchSessionId = searchSessionManager.getNextSearchSessionId(); + updateSearchSourceHelper(); - $scope - .updateDataSource() - .then(setupVisualization) - .then(function () { - $scope.fetchStatus = fetchStatuses.LOADING; - logInspectorRequest({ searchSessionId }); - return $scope.volatileSearchSource.fetch({ - abortSignal: abortController.signal, - sessionId: searchSessionId, - }); + $scope.opts.chartAggConfigs = applyAggsToSearchSource( + getTimeField() && !$scope.state.hideChart, + volatileSearchSource, + $scope.state.interval, + $scope.indexPattern, + data + ); + + $scope.fetchStatus = fetchStatuses.LOADING; + $scope.resultState = getResultState($scope.fetchStatus, $scope.rows); + logInspectorRequest({ searchSessionId }); + return $scope.volatileSearchSource + .fetch({ + abortSignal: abortController.signal, + sessionId: searchSessionId, }) .then(onResults) .catch((error) => { @@ -565,40 +431,14 @@ function discoverController($route, $scope, Promise) { $scope.fetchStatus = fetchStatuses.NO_RESULTS; $scope.fetchError = error; - data.search.showError(error); + }) + .finally(() => { + $scope.resultState = getResultState($scope.fetchStatus, $scope.rows); + $scope.$apply(); }); }; - function getDimensions(aggs, timeRange) { - const [metric, agg] = aggs; - agg.params.timeRange = timeRange; - const bounds = agg.params.timeRange ? timefilter.calculateBounds(agg.params.timeRange) : null; - agg.buckets.setBounds(bounds); - - const { esUnit, esValue } = agg.buckets.getInterval(); - return { - x: { - accessor: 0, - label: agg.makeLabel(), - format: agg.toSerializedFieldFormat(), - params: { - date: true, - interval: moment.duration(esValue, esUnit), - intervalESValue: esValue, - intervalESUnit: esUnit, - format: agg.buckets.getScaledDateFormat(), - bounds: agg.buckets.getBounds(), - }, - }, - y: { - accessor: 1, - format: metric.toSerializedFieldFormat(), - label: metric.makeLabel(), - }, - }; - } - function onResults(resp) { inspectorRequest .stats(getResponseInspectorStats(resp, $scope.volatileSearchSource)) @@ -607,11 +447,10 @@ function discoverController($route, $scope, Promise) { if (getTimeField() && !$scope.state.hideChart) { const tabifiedData = tabifyAggResponse($scope.opts.chartAggConfigs, resp); $scope.volatileSearchSource.rawResponse = resp; - $scope.histogramData = discoverResponseHandler( - tabifiedData, - getDimensions($scope.opts.chartAggConfigs.aggs, $scope.timeRange) - ); - $scope.updateTime(); + const dimensions = getDimensions($scope.opts.chartAggConfigs, data); + if (dimensions) { + $scope.histogramData = discoverResponseHandler(tabifiedData, dimensions); + } } $scope.hits = resp.hits.total; @@ -640,15 +479,6 @@ function discoverController($route, $scope, Promise) { }); } - $scope.updateTime = function () { - const { from, to } = timefilter.getTime(); - // this is the timerange for the histogram, should be refactored - $scope.timeRange = { - from: dateMath.parse(from), - to: dateMath.parse(to, { roundUp: true }), - }; - }; - $scope.resetQuery = function () { history.push( $route.current.params.id ? `/view/${encodeURIComponent($route.current.params.id)}` : '/' @@ -656,88 +486,39 @@ function discoverController($route, $scope, Promise) { $route.reload(); }; - $scope.onSkipBottomButtonClick = async () => { - // show all the Rows - $scope.minimumVisibleRows = $scope.hits; - - // delay scrolling to after the rows have been rendered - const bottomMarker = document.getElementById('discoverBottomMarker'); - const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - - while ($scope.rows.length !== document.getElementsByClassName('kbnDocTable__row').length) { - await wait(50); - } - bottomMarker.focus(); - await wait(50); - bottomMarker.blur(); - }; - $scope.newQuery = function () { history.push('/'); }; - const showUnmappedFields = $scope.useNewFieldsApi; - $scope.unmappedFieldsConfig = { showUnmappedFields, }; - $scope.updateDataSource = () => { - const { indexPattern, useNewFieldsApi } = $scope; - const { columns, sort } = $scope.state; - updateSearchSource({ - persistentSearchSource, - volatileSearchSource: $scope.volatileSearchSource, - indexPattern, - services, - sort, - columns, - useNewFieldsApi, - showUnmappedFields, - }); - return Promise.resolve(); - }; - - async function setupVisualization() { - // If no timefield has been specified we don't create a histogram of messages - if (!getTimeField() || $scope.state.hideChart) { - if ($scope.volatileSearchSource.getField('aggs')) { - // cleanup aggs field in case it was set before - $scope.volatileSearchSource.removeField('aggs'); - } - return; - } - const { interval: histogramInterval } = $scope.state; + const fetch$ = merge( + refetch$, + filterManager.getFetches$(), + timefilter.getFetch$(), + timefilter.getAutoRefreshFetch$(), + data.query.queryString.getUpdates$(), + searchSessionManager.newSearchSessionIdFromURL$ + ).pipe(debounceTime(100)); - const visStateAggs = [ - { - type: 'count', - schema: 'metric', - }, + subscriptions.add( + subscribeWithScope( + $scope, + fetch$, { - type: 'date_histogram', - schema: 'segment', - params: { - field: getTimeField(), - interval: histogramInterval, - timeRange: timefilter.getTime(), - }, + next: $scope.fetch, }, - ]; - $scope.opts.chartAggConfigs = data.search.aggs.createAggConfigs( - $scope.indexPattern, - visStateAggs - ); - - $scope.volatileSearchSource.setField('aggs', function () { - if (!$scope.opts.chartAggConfigs) return; - return $scope.opts.chartAggConfigs.toDsl(); - }); - } - - addHelpMenuToAppChrome(chrome); + (error) => addFatalError(core.fatalErrors, error) + ) + ); - init(); - // Propagate current app state to url, then start syncing - replaceUrlAppState().then(() => startStateSync()); + // Propagate current app state to url, then start syncing and fetching + replaceUrlAppState().then(() => { + startStateSync(); + if (shouldSearchOnPageLoad()) { + refetch$.next(); + } + }); } diff --git a/src/plugins/discover/public/application/angular/discover_legacy.html b/src/plugins/discover/public/application/angular/discover_legacy.html index a01f285b1a150..f14800f81d08e 100644 --- a/src/plugins/discover/public/application/angular/discover_legacy.html +++ b/src/plugins/discover/public/application/angular/discover_legacy.html @@ -7,15 +7,12 @@ histogram-data="histogramData" hits="hits" index-pattern="indexPattern" - minimum-visible-rows="minimumVisibleRows" - on-skip-bottom-button-click="onSkipBottomButtonClick" opts="opts" reset-query="resetQuery" result-state="resultState" rows="rows" search-source="volatileSearchSource" state="state" - time-range="timeRange" top-nav-menu="topNavMenu" use-new-fields-api="useNewFieldsApi" unmapped-fields-config="unmappedFieldsConfig" diff --git a/src/plugins/discover/public/application/angular/doc_table/create_doc_table_react.tsx b/src/plugins/discover/public/application/angular/doc_table/create_doc_table_react.tsx index 4dfd821abd430..0202f88e0e902 100644 --- a/src/plugins/discover/public/application/angular/doc_table/create_doc_table_react.tsx +++ b/src/plugins/discover/public/application/angular/doc_table/create_doc_table_react.tsx @@ -8,12 +8,13 @@ import angular, { auto, ICompileService, IScope } from 'angular'; import { render } from 'react-dom'; -import React, { useRef, useEffect } from 'react'; +import React, { useRef, useEffect, useState, useCallback } from 'react'; import type { estypes } from '@elastic/elasticsearch'; import { EuiButtonEmpty } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { getServices, IIndexPattern } from '../../../kibana_services'; import { IndexPatternField } from '../../../../../data/common/index_patterns'; +import { SkipBottomButton } from '../../components/skip_bottom_button'; export interface DocTableLegacyProps { columns: string[]; @@ -98,18 +99,42 @@ function getRenderFn(domNode: Element, props: any) { export function DocTableLegacy(renderProps: DocTableLegacyProps) { const ref = useRef(null); const scope = useRef(); + const [rows, setRows] = useState(renderProps.rows); + const [minimumVisibleRows, setMinimumVisibleRows] = useState(50); + const onSkipBottomButtonClick = useCallback(async () => { + // delay scrolling to after the rows have been rendered + const bottomMarker = document.getElementById('discoverBottomMarker'); + const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + // show all the rows + setMinimumVisibleRows(renderProps.rows.length); + + while (renderProps.rows.length !== document.getElementsByClassName('kbnDocTable__row').length) { + await wait(50); + } + bottomMarker!.focus(); + await wait(50); + bottomMarker!.blur(); + }, [setMinimumVisibleRows, renderProps.rows]); + + useEffect(() => { + if (minimumVisibleRows > 50) { + setMinimumVisibleRows(50); + } + setRows(renderProps.rows); + }, [renderProps.rows, minimumVisibleRows, setMinimumVisibleRows]); useEffect(() => { if (ref && ref.current && !scope.current) { - const fn = getRenderFn(ref.current, renderProps); + const fn = getRenderFn(ref.current, { ...renderProps, rows, minimumVisibleRows }); fn().then((newScope) => { scope.current = newScope; }); } else if (scope && scope.current) { - scope.current.renderProps = renderProps; + scope.current.renderProps = { ...renderProps, rows, minimumVisibleRows }; scope.current.$apply(); } - }, [renderProps]); + }, [renderProps, minimumVisibleRows, rows]); + useEffect(() => { return () => { if (scope.current) { @@ -119,6 +144,7 @@ export function DocTableLegacy(renderProps: DocTableLegacyProps) { }, []); return (
+
{renderProps.rows.length === renderProps.sampleSize ? (
- +
diff --git a/src/plugins/discover/public/application/angular/doc_table/lib/get_sort.ts b/src/plugins/discover/public/application/angular/doc_table/lib/get_sort.ts index 4838a4019357c..4b16c1aa3dcc6 100644 --- a/src/plugins/discover/public/application/angular/doc_table/lib/get_sort.ts +++ b/src/plugins/discover/public/application/angular/doc_table/lib/get_sort.ts @@ -14,9 +14,9 @@ export type SortPairArr = [string, string]; export type SortPair = SortPairArr | SortPairObj; export type SortInput = SortPair | SortPair[]; -export function isSortable(fieldName: string, indexPattern: IndexPattern) { +export function isSortable(fieldName: string, indexPattern: IndexPattern): boolean { const field = indexPattern.getFieldByName(fieldName); - return field && field.sortable; + return !!(field && field.sortable); } function createSortObject( diff --git a/src/plugins/discover/public/application/components/constants.ts b/src/plugins/discover/public/application/components/constants.ts new file mode 100644 index 0000000000000..42845e83b7435 --- /dev/null +++ b/src/plugins/discover/public/application/components/constants.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const fetchStatuses = { + UNINITIALIZED: 'uninitialized', + LOADING: 'loading', + COMPLETE: 'complete', + ERROR: 'error', +}; diff --git a/src/plugins/discover/public/application/components/create_discover_directive.ts b/src/plugins/discover/public/application/components/create_discover_directive.ts index 8d1360aeaddad..5abf87fdfbc08 100644 --- a/src/plugins/discover/public/application/components/create_discover_directive.ts +++ b/src/plugins/discover/public/application/components/create_discover_directive.ts @@ -16,8 +16,6 @@ export function createDiscoverDirective(reactDirective: any) { ['histogramData', { watchDepth: 'reference' }], ['hits', { watchDepth: 'reference' }], ['indexPattern', { watchDepth: 'reference' }], - ['minimumVisibleRows', { watchDepth: 'reference' }], - ['onSkipBottomButtonClick', { watchDepth: 'reference' }], ['opts', { watchDepth: 'reference' }], ['resetQuery', { watchDepth: 'reference' }], ['resultState', { watchDepth: 'reference' }], @@ -26,7 +24,6 @@ export function createDiscoverDirective(reactDirective: any) { ['searchSource', { watchDepth: 'reference' }], ['showSaveQuery', { watchDepth: 'reference' }], ['state', { watchDepth: 'reference' }], - ['timeRange', { watchDepth: 'reference' }], ['topNavMenu', { watchDepth: 'reference' }], ['updateQuery', { watchDepth: 'reference' }], ['updateSavedQueryId', { watchDepth: 'reference' }], diff --git a/src/plugins/discover/public/application/components/discover.tsx b/src/plugins/discover/public/application/components/discover.tsx index 056581e30b4d6..9615a1c10ea8e 100644 --- a/src/plugins/discover/public/application/components/discover.tsx +++ b/src/plugins/discover/public/application/components/discover.tsx @@ -6,7 +6,7 @@ * Side Public License, v 1. */ import './discover.scss'; -import React, { useState, useRef, useMemo, useCallback } from 'react'; +import React, { useState, useRef, useMemo, useCallback, useEffect } from 'react'; import { EuiButtonEmpty, EuiButtonIcon, @@ -30,7 +30,6 @@ import { DiscoverHistogram, DiscoverUninitialized } from '../angular/directives' import { DiscoverNoResults } from './no_results'; import { LoadingSpinner } from './loading_spinner/loading_spinner'; import { DocTableLegacy } from '../angular/doc_table/create_doc_table_react'; -import { SkipBottomButton } from './skip_bottom_button'; import { esFilters, IndexPatternField, search } from '../../../../data/public'; import { DiscoverSidebarResponsive } from './sidebar'; import { DiscoverProps } from './types'; @@ -42,11 +41,15 @@ import { DocViewFilterFn } from '../doc_views/doc_views_types'; import { DiscoverGrid } from './discover_grid/discover_grid'; import { DiscoverTopNav } from './discover_topnav'; import { ElasticSearchHit } from '../doc_views/doc_views_types'; +import { setBreadcrumbsTitle } from '../helpers/breadcrumbs'; +import { addHelpMenuToAppChrome } from './help_menu/help_menu_util'; const DocTableLegacyMemoized = React.memo(DocTableLegacy); const SidebarMemoized = React.memo(DiscoverSidebarResponsive); const DataGridMemoized = React.memo(DiscoverGrid); const TopNavMemoized = React.memo(DiscoverTopNav); +const TimechartHeaderMemoized = React.memo(TimechartHeader); +const DiscoverHistogramMemoized = React.memo(DiscoverHistogram); export function Discover({ fetch, @@ -58,14 +61,12 @@ export function Discover({ hits, indexPattern, minimumVisibleRows, - onSkipBottomButtonClick, opts, resetQuery, resultState, rows, searchSource, state, - timeRange, unmappedFieldsConfig, }: DiscoverProps) { const [expandedDoc, setExpandedDoc] = useState(undefined); @@ -81,13 +82,16 @@ export function Discover({ }, [state, opts]); const hideChart = useMemo(() => state.hideChart, [state]); const { savedSearch, indexPatternList, config, services, data, setAppState } = opts; - const { trackUiMetric, capabilities, indexPatterns } = services; + const { trackUiMetric, capabilities, indexPatterns, chrome, docLinks } = services; + const [isSidebarClosed, setIsSidebarClosed] = useState(false); - const bucketAggConfig = opts.chartAggConfigs?.aggs[1]; - const bucketInterval = - bucketAggConfig && search.aggs.isDateHistogramBucketAggConfig(bucketAggConfig) + const bucketInterval = useMemo(() => { + const bucketAggConfig = opts.chartAggConfigs?.aggs[1]; + return bucketAggConfig && search.aggs.isDateHistogramBucketAggConfig(bucketAggConfig) ? bucketAggConfig.buckets?.getInterval() : undefined; + }, [opts.chartAggConfigs]); + const contentCentered = resultState === 'uninitialized'; const isLegacy = services.uiSettings.get('doc_table:legacy'); const useNewFieldsApi = !services.uiSettings.get(SEARCH_FIELDS_FROM_SOURCE); @@ -101,6 +105,14 @@ export function Discover({ [opts] ); + useEffect(() => { + const pageTitleSuffix = savedSearch.id && savedSearch.title ? `: ${savedSearch.title}` : ''; + chrome.docTitle.change(`Discover${pageTitleSuffix}`); + + setBreadcrumbsTitle(savedSearch, chrome); + addHelpMenuToAppChrome(chrome, docLinks); + }, [savedSearch, chrome, docLinks]); + const { onAddColumn, onRemoveColumn, onMoveColumn, onSetColumns } = useMemo( () => getStateColumnActions({ @@ -293,9 +305,9 @@ export function Discover({ {!hideChart && ( - )} - {isLegacy && } {!hideChart && opts.timefield && ( @@ -342,7 +353,7 @@ export function Discover({ className={isLegacy ? 'dscHistogram' : 'dscHistogramGrid'} data-test-subj="discoverChart" > - diff --git a/src/plugins/discover/public/application/components/discover_grid/discover_grid_columns.test.tsx b/src/plugins/discover/public/application/components/discover_grid/discover_grid_columns.test.tsx index 1a721a400803e..93b5bf8fde0c1 100644 --- a/src/plugins/discover/public/application/components/discover_grid/discover_grid_columns.test.tsx +++ b/src/plugins/discover/public/application/components/discover_grid/discover_grid_columns.test.tsx @@ -64,11 +64,14 @@ describe('Discover grid columns ', function () { "showMoveLeft": false, "showMoveRight": false, }, - "cellActions": undefined, + "cellActions": Array [ + [Function], + [Function], + ], "display": undefined, "id": "extension", "isSortable": false, - "schema": "kibana-json", + "schema": "string", }, Object { "actions": Object { @@ -80,7 +83,7 @@ describe('Discover grid columns ', function () { "display": undefined, "id": "message", "isSortable": false, - "schema": "kibana-json", + "schema": "string", }, ] `); @@ -101,12 +104,15 @@ describe('Discover grid columns ', function () { "showMoveLeft": true, "showMoveRight": true, }, - "cellActions": undefined, + "cellActions": Array [ + [Function], + [Function], + ], "display": "Time (timestamp)", "id": "timestamp", "initialWidth": 180, - "isSortable": false, - "schema": "kibana-json", + "isSortable": true, + "schema": "datetime", }, Object { "actions": Object { @@ -117,11 +123,14 @@ describe('Discover grid columns ', function () { "showMoveLeft": true, "showMoveRight": true, }, - "cellActions": undefined, + "cellActions": Array [ + [Function], + [Function], + ], "display": undefined, "id": "extension", "isSortable": false, - "schema": "kibana-json", + "schema": "string", }, Object { "actions": Object { @@ -136,7 +145,7 @@ describe('Discover grid columns ', function () { "display": undefined, "id": "message", "isSortable": false, - "schema": "kibana-json", + "schema": "string", }, ] `); diff --git a/src/plugins/discover/public/application/components/help_menu/help_menu_util.js b/src/plugins/discover/public/application/components/help_menu/help_menu_util.ts similarity index 81% rename from src/plugins/discover/public/application/components/help_menu/help_menu_util.js rename to src/plugins/discover/public/application/components/help_menu/help_menu_util.ts index 1a6815b40b581..d0d5cfde1fe06 100644 --- a/src/plugins/discover/public/application/components/help_menu/help_menu_util.js +++ b/src/plugins/discover/public/application/components/help_menu/help_menu_util.ts @@ -7,10 +7,9 @@ */ import { i18n } from '@kbn/i18n'; -import { getServices } from '../../../kibana_services'; -const { docLinks } = getServices(); +import { ChromeStart, DocLinksStart } from 'kibana/public'; -export function addHelpMenuToAppChrome(chrome) { +export function addHelpMenuToAppChrome(chrome: ChromeStart, docLinks: DocLinksStart) { chrome.setHelpExtension({ appName: i18n.translate('discover.helpMenu.appName', { defaultMessage: 'Discover', diff --git a/src/plugins/discover/public/application/components/histogram/apply_aggs_to_search_source.test.ts b/src/plugins/discover/public/application/components/histogram/apply_aggs_to_search_source.test.ts new file mode 100644 index 0000000000000..29c93886ebba3 --- /dev/null +++ b/src/plugins/discover/public/application/components/histogram/apply_aggs_to_search_source.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { indexPatternWithTimefieldMock } from '../../../__mocks__/index_pattern_with_timefield'; +import { SearchSource } from '../../../../../data/public'; +import { dataPluginMock } from '../../../../../data/public/mocks'; +import { applyAggsToSearchSource } from './apply_aggs_to_search_source'; + +describe('applyAggsToSearchSource', () => { + test('enabled = true', () => { + const indexPattern = indexPatternWithTimefieldMock; + const setField = jest.fn(); + const searchSource = ({ + setField, + removeField: jest.fn(), + } as unknown) as SearchSource; + + const dataMock = dataPluginMock.createStartContract(); + + const aggsConfig = applyAggsToSearchSource(true, searchSource, 'auto', indexPattern, dataMock); + + expect(aggsConfig!.aggs).toMatchInlineSnapshot(` + Array [ + Object { + "enabled": true, + "id": "1", + "params": Object {}, + "schema": "metric", + "type": "count", + }, + Object { + "enabled": true, + "id": "2", + "params": Object { + "drop_partials": false, + "extended_bounds": Object {}, + "field": "timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "useNormalizedEsInterval": true, + }, + "schema": "segment", + "type": "date_histogram", + }, + ] + `); + + expect(setField).toHaveBeenCalledWith('aggs', expect.any(Function)); + const dslFn = setField.mock.calls[0][1]; + expect(dslFn()).toMatchInlineSnapshot(` + Object { + "2": Object { + "date_histogram": Object { + "field": "timestamp", + "min_doc_count": 1, + "time_zone": "America/New_York", + }, + }, + } + `); + }); + + test('enabled = false', () => { + const indexPattern = indexPatternWithTimefieldMock; + const setField = jest.fn(); + const getField = jest.fn(() => { + return true; + }); + const removeField = jest.fn(); + const searchSource = ({ + getField, + setField, + removeField, + } as unknown) as SearchSource; + + const dataMock = dataPluginMock.createStartContract(); + + const aggsConfig = applyAggsToSearchSource(false, searchSource, 'auto', indexPattern, dataMock); + expect(aggsConfig).toBeFalsy(); + expect(getField).toHaveBeenCalledWith('aggs'); + expect(removeField).toHaveBeenCalledWith('aggs'); + }); +}); diff --git a/src/plugins/discover/public/application/components/histogram/apply_aggs_to_search_source.ts b/src/plugins/discover/public/application/components/histogram/apply_aggs_to_search_source.ts new file mode 100644 index 0000000000000..c5fb366f81c8c --- /dev/null +++ b/src/plugins/discover/public/application/components/histogram/apply_aggs_to_search_source.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { IndexPattern, SearchSource } from '../../../../../data/common'; +import { DataPublicPluginStart } from '../../../../../data/public'; + +/** + * Helper function to apply or remove aggregations to a given search source used for gaining data + * for Discover's histogram vis + */ +export function applyAggsToSearchSource( + enabled: boolean, + searchSource: SearchSource, + histogramInterval: string, + indexPattern: IndexPattern, + data: DataPublicPluginStart +) { + if (!enabled) { + if (searchSource.getField('aggs')) { + // clean up fields in case it was set before + searchSource.removeField('aggs'); + } + return; + } + const visStateAggs = [ + { + type: 'count', + schema: 'metric', + }, + { + type: 'date_histogram', + schema: 'segment', + params: { + field: indexPattern.timeFieldName!, + interval: histogramInterval, + timeRange: data.query.timefilter.timefilter.getTime(), + }, + }, + ]; + const chartAggConfigs = data.search.aggs.createAggConfigs(indexPattern, visStateAggs); + + searchSource.setField('aggs', function () { + return chartAggConfigs.toDsl(); + }); + return chartAggConfigs; +} diff --git a/src/plugins/discover/public/application/components/histogram/get_dimensions.test.ts b/src/plugins/discover/public/application/components/histogram/get_dimensions.test.ts new file mode 100644 index 0000000000000..ad7031f331992 --- /dev/null +++ b/src/plugins/discover/public/application/components/histogram/get_dimensions.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { dataPluginMock } from '../../../../../data/public/mocks'; + +import { getDimensions } from './get_dimensions'; +import { indexPatternWithTimefieldMock } from '../../../__mocks__/index_pattern_with_timefield'; +import { SearchSource } from '../../../../../data/common/search/search_source'; +import { applyAggsToSearchSource } from './apply_aggs_to_search_source'; +import { calculateBounds } from '../../../../../data/common/query/timefilter'; + +test('getDimensions', () => { + const indexPattern = indexPatternWithTimefieldMock; + const setField = jest.fn(); + const searchSource = ({ + setField, + removeField: jest.fn(), + } as unknown) as SearchSource; + + const dataMock = dataPluginMock.createStartContract(); + dataMock.query.timefilter.timefilter.getTime = () => { + return { from: 'now-30y', to: 'now' }; + }; + dataMock.query.timefilter.timefilter.calculateBounds = (timeRange) => { + return calculateBounds(timeRange); + }; + + const aggsConfig = applyAggsToSearchSource(true, searchSource, 'auto', indexPattern, dataMock); + const actual = getDimensions(aggsConfig!, dataMock); + expect(actual).toMatchInlineSnapshot(` + Object { + "x": Object { + "accessor": 0, + "format": Object { + "id": "date", + "params": Object { + "pattern": "HH:mm:ss.SSS", + }, + }, + "label": "timestamp per 0 milliseconds", + "params": Object { + "bounds": undefined, + "date": true, + "format": "HH:mm:ss.SSS", + "interval": "P365D", + "intervalESUnit": "d", + "intervalESValue": 365, + }, + }, + "y": Object { + "accessor": 1, + "format": Object { + "id": "number", + }, + "label": "Count", + }, + } + `); +}); diff --git a/src/plugins/discover/public/application/components/histogram/get_dimensions.ts b/src/plugins/discover/public/application/components/histogram/get_dimensions.ts new file mode 100644 index 0000000000000..6743c1c8431b9 --- /dev/null +++ b/src/plugins/discover/public/application/components/histogram/get_dimensions.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import moment from 'moment'; +import dateMath from '@elastic/datemath'; +import { IAggConfigs, TimeRangeBounds } from '../../../../../data/common'; +import { DataPublicPluginStart, search } from '../../../../../data/public'; + +export function getDimensions(aggs: IAggConfigs, data: DataPublicPluginStart) { + const [metric, agg] = aggs.aggs; + const { from, to } = data.query.timefilter.timefilter.getTime(); + agg.params.timeRange = { + from: dateMath.parse(from), + to: dateMath.parse(to, { roundUp: true }), + }; + const bounds = agg.params.timeRange + ? data.query.timefilter.timefilter.calculateBounds(agg.params.timeRange) + : null; + const buckets = search.aggs.isDateHistogramBucketAggConfig(agg) ? agg.buckets : undefined; + + if (!buckets) { + return; + } + + buckets.setBounds(bounds as TimeRangeBounds); + + const { esUnit, esValue } = buckets.getInterval(); + return { + x: { + accessor: 0, + label: agg.makeLabel(), + format: agg.toSerializedFieldFormat(), + params: { + date: true, + interval: moment.duration(esValue, esUnit), + intervalESValue: esValue, + intervalESUnit: esUnit, + format: buckets.getScaledDateFormat(), + bounds: buckets.getBounds(), + }, + }, + y: { + accessor: 1, + format: metric.toSerializedFieldFormat(), + label: metric.makeLabel(), + }, + }; +} diff --git a/src/plugins/discover/public/application/components/histogram/index.ts b/src/plugins/discover/public/application/components/histogram/index.ts new file mode 100644 index 0000000000000..4af75de0a029d --- /dev/null +++ b/src/plugins/discover/public/application/components/histogram/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { applyAggsToSearchSource } from './apply_aggs_to_search_source'; +export { getDimensions } from './get_dimensions'; diff --git a/src/plugins/discover/public/application/components/timechart_header/timechart_header.test.tsx b/src/plugins/discover/public/application/components/timechart_header/timechart_header.test.tsx index ff8f14115e492..74836711373b2 100644 --- a/src/plugins/discover/public/application/components/timechart_header/timechart_header.test.tsx +++ b/src/plugins/discover/public/application/components/timechart_header/timechart_header.test.tsx @@ -12,6 +12,7 @@ import { ReactWrapper } from 'enzyme'; import { TimechartHeader, TimechartHeaderProps } from './timechart_header'; import { EuiIconTip } from '@elastic/eui'; import { findTestSubject } from '@elastic/eui/lib/test'; +import { DataPublicPluginStart } from '../../../../../data/public'; describe('timechart header', function () { let props: TimechartHeaderProps; @@ -19,10 +20,18 @@ describe('timechart header', function () { beforeAll(() => { props = { - timeRange: { - from: 'May 14, 2020 @ 11:05:13.590', - to: 'May 14, 2020 @ 11:20:13.590', - }, + data: { + query: { + timefilter: { + timefilter: { + getTime: () => { + return { from: '2020-05-14T11:05:13.590', to: '2020-05-14T11:20:13.590' }; + }, + }, + }, + }, + } as DataPublicPluginStart, + dateFormat: 'MMM D, YYYY @ HH:mm:ss.SSS', stateInterval: 's', options: [ { diff --git a/src/plugins/discover/public/application/components/timechart_header/timechart_header.tsx b/src/plugins/discover/public/application/components/timechart_header/timechart_header.tsx index 0379059b80e58..a2fc17e05a203 100644 --- a/src/plugins/discover/public/application/components/timechart_header/timechart_header.tsx +++ b/src/plugins/discover/public/application/components/timechart_header/timechart_header.tsx @@ -15,9 +15,11 @@ import { EuiSelect, EuiIconTip, } from '@elastic/eui'; +import moment from 'moment'; import { i18n } from '@kbn/i18n'; +import dateMath from '@elastic/datemath'; +import { DataPublicPluginStart } from '../../../../../data/public'; import './timechart_header.scss'; -import moment from 'moment'; export interface TimechartHeaderProps { /** @@ -32,13 +34,7 @@ export interface TimechartHeaderProps { description?: string; scale?: number; }; - /** - * Range of dates to be displayed - */ - timeRange?: { - from: string; - to: string; - }; + data: DataPublicPluginStart; /** * Interval Options */ @@ -56,21 +52,27 @@ export interface TimechartHeaderProps { export function TimechartHeader({ bucketInterval, dateFormat, - timeRange, + data, options, onChangeInterval, stateInterval, }: TimechartHeaderProps) { + const { timefilter } = data.query.timefilter; + const { from, to } = timefilter.getTime(); + const timeRange = { + from: dateMath.parse(from), + to: dateMath.parse(to, { roundUp: true }), + }; const [interval, setInterval] = useState(stateInterval); const toMoment = useCallback( - (datetime: string) => { + (datetime: moment.Moment | undefined) => { if (!datetime) { return ''; } if (!dateFormat) { - return datetime; + return String(datetime); } - return moment(datetime).format(dateFormat); + return datetime.format(dateFormat); }, [dateFormat] ); diff --git a/src/plugins/discover/public/application/components/types.ts b/src/plugins/discover/public/application/components/types.ts index db1cd89422454..23a3cc9a9bc74 100644 --- a/src/plugins/discover/public/application/components/types.ts +++ b/src/plugins/discover/public/application/components/types.ts @@ -158,10 +158,6 @@ export interface DiscoverProps { * Current app state of URL */ state: AppState; - /** - * Currently selected time range - */ - timeRange?: { from: string; to: string }; /** * An object containing properties for unmapped fields behavior */ diff --git a/src/plugins/discover/public/application/helpers/get_result_state.test.ts b/src/plugins/discover/public/application/helpers/get_result_state.test.ts new file mode 100644 index 0000000000000..98e2b854ca5ab --- /dev/null +++ b/src/plugins/discover/public/application/helpers/get_result_state.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { getResultState, resultStatuses } from './get_result_state'; +import { fetchStatuses } from '../components/constants'; +import { ElasticSearchHit } from '../doc_views/doc_views_types'; + +describe('getResultState', () => { + test('fetching uninitialized', () => { + const actual = getResultState(fetchStatuses.UNINITIALIZED, []); + expect(actual).toBe(resultStatuses.UNINITIALIZED); + }); + + test('fetching complete with no records', () => { + const actual = getResultState(fetchStatuses.COMPLETE, []); + expect(actual).toBe(resultStatuses.NO_RESULTS); + }); + + test('fetching ongoing aka loading', () => { + const actual = getResultState(fetchStatuses.LOADING, []); + expect(actual).toBe(resultStatuses.LOADING); + }); + + test('fetching ready', () => { + const record = ({ _id: 123 } as unknown) as ElasticSearchHit; + const actual = getResultState(fetchStatuses.COMPLETE, [record]); + expect(actual).toBe(resultStatuses.READY); + }); + + test('re-fetching after already data is available', () => { + const record = ({ _id: 123 } as unknown) as ElasticSearchHit; + const actual = getResultState(fetchStatuses.LOADING, [record]); + expect(actual).toBe(resultStatuses.READY); + }); + + test('after a fetch error when data was successfully fetched before ', () => { + const record = ({ _id: 123 } as unknown) as ElasticSearchHit; + const actual = getResultState(fetchStatuses.ERROR, [record]); + expect(actual).toBe(resultStatuses.READY); + }); +}); diff --git a/src/plugins/discover/public/application/helpers/get_result_state.ts b/src/plugins/discover/public/application/helpers/get_result_state.ts new file mode 100644 index 0000000000000..6f69832f369fd --- /dev/null +++ b/src/plugins/discover/public/application/helpers/get_result_state.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { ElasticSearchHit } from '../doc_views/doc_views_types'; +import { fetchStatuses } from '../components/constants'; + +export const resultStatuses = { + UNINITIALIZED: 'uninitialized', + LOADING: 'loading', // initial data load + READY: 'ready', // results came back + NO_RESULTS: 'none', // no results came back +}; + +/** + * Returns the current state of the result, depends on fetchStatus and the given fetched rows + * Determines what is displayed in Discover main view (loading view, data view, empty data view, ...) + */ +export function getResultState(fetchStatus: string, rows: ElasticSearchHit[]) { + if (fetchStatus === fetchStatuses.UNINITIALIZED) { + return resultStatuses.UNINITIALIZED; + } + + const rowsEmpty = !Array.isArray(rows) || rows.length === 0; + if (rowsEmpty && fetchStatus === fetchStatuses.LOADING) return resultStatuses.LOADING; + else if (!rowsEmpty) return resultStatuses.READY; + else return resultStatuses.NO_RESULTS; +} diff --git a/src/plugins/discover/public/application/helpers/get_state_defaults.test.ts b/src/plugins/discover/public/application/helpers/get_state_defaults.test.ts new file mode 100644 index 0000000000000..7ce5b9286c775 --- /dev/null +++ b/src/plugins/discover/public/application/helpers/get_state_defaults.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getStateDefaults } from './get_state_defaults'; +import { createSearchSourceMock, dataPluginMock } from '../../../../data/public/mocks'; +import { uiSettingsMock } from '../../__mocks__/ui_settings'; +import { indexPatternWithTimefieldMock } from '../../__mocks__/index_pattern_with_timefield'; +import { savedSearchMock } from '../../__mocks__/saved_search'; +import { indexPatternMock } from '../../__mocks__/index_pattern'; + +describe('getStateDefaults', () => { + test('index pattern with timefield', () => { + const actual = getStateDefaults({ + config: uiSettingsMock, + data: dataPluginMock.createStartContract(), + indexPattern: indexPatternWithTimefieldMock, + savedSearch: savedSearchMock, + searchSource: createSearchSourceMock({ index: indexPatternWithTimefieldMock }), + }); + expect(actual).toMatchInlineSnapshot(` + Object { + "columns": Array [ + "default_column", + ], + "filters": undefined, + "index": "index-pattern-with-timefield-id", + "interval": "auto", + "query": undefined, + "sort": Array [ + Array [ + "timestamp", + "desc", + ], + ], + } + `); + }); + + test('index pattern without timefield', () => { + const actual = getStateDefaults({ + config: uiSettingsMock, + data: dataPluginMock.createStartContract(), + indexPattern: indexPatternMock, + savedSearch: savedSearchMock, + searchSource: createSearchSourceMock({ index: indexPatternMock }), + }); + expect(actual).toMatchInlineSnapshot(` + Object { + "columns": Array [ + "default_column", + ], + "filters": undefined, + "index": "the-index-pattern-id", + "interval": "auto", + "query": undefined, + "sort": Array [], + } + `); + }); +}); diff --git a/src/plugins/discover/public/application/helpers/get_state_defaults.ts b/src/plugins/discover/public/application/helpers/get_state_defaults.ts new file mode 100644 index 0000000000000..3e012a1f85fd6 --- /dev/null +++ b/src/plugins/discover/public/application/helpers/get_state_defaults.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { cloneDeep } from 'lodash'; +import { IUiSettingsClient } from 'kibana/public'; +import { DEFAULT_COLUMNS_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common'; +import { getSortArray } from '../angular/doc_table'; +import { getDefaultSort } from '../angular/doc_table/lib/get_default_sort'; +import { SavedSearch } from '../../saved_searches'; +import { SearchSource } from '../../../../data/common/search/search_source'; +import { DataPublicPluginStart, IndexPattern } from '../../../../data/public'; + +import { AppState } from '../angular/discover_state'; + +function getDefaultColumns(savedSearch: SavedSearch, config: IUiSettingsClient) { + if (savedSearch.columns && savedSearch.columns.length > 0) { + return [...savedSearch.columns]; + } + return [...config.get(DEFAULT_COLUMNS_SETTING)]; +} + +export function getStateDefaults({ + config, + data, + indexPattern, + savedSearch, + searchSource, +}: { + config: IUiSettingsClient; + data: DataPublicPluginStart; + indexPattern: IndexPattern; + savedSearch: SavedSearch; + searchSource: SearchSource; +}) { + const query = searchSource.getField('query') || data.query.queryString.getDefaultQuery(); + const sort = getSortArray(savedSearch.sort, indexPattern); + const columns = getDefaultColumns(savedSearch, config); + + const defaultState = { + query, + sort: !sort.length + ? getDefaultSort(indexPattern, config.get(SORT_DEFAULT_ORDER_SETTING, 'desc')) + : sort, + columns, + index: indexPattern.id, + interval: 'auto', + filters: cloneDeep(searchSource.getOwnField('filter')), + } as AppState; + if (savedSearch.grid) { + defaultState.grid = savedSearch.grid; + } + if (savedSearch.hideChart) { + defaultState.hideChart = savedSearch.hideChart; + } + + return defaultState; +} diff --git a/src/setup_node_env/ensure_node_preserve_symlinks.js b/src/setup_node_env/ensure_node_preserve_symlinks.js index 0d72ec85e6c87..826244c4829fc 100644 --- a/src/setup_node_env/ensure_node_preserve_symlinks.js +++ b/src/setup_node_env/ensure_node_preserve_symlinks.js @@ -9,10 +9,51 @@ (function () { var cp = require('child_process'); + var calculateInspectPortOnExecArgv = function (processExecArgv) { + var execArgv = [].concat(processExecArgv); + + if (execArgv.length === 0) { + return execArgv; + } + + var inspectFlagIndex = execArgv.reverse().findIndex(function (flag) { + return flag.startsWith('--inspect'); + }); + + if (inspectFlagIndex !== -1) { + var inspectFlag; + var inspectPortCounter = 9230; + var argv = execArgv[inspectFlagIndex]; + + if (argv.includes('=')) { + // --inspect=port + var argvSplit = argv.split('='); + var flag = argvSplit[0]; + var port = argvSplit[1]; + inspectFlag = flag; + inspectPortCounter = Number.parseInt(port, 10) + 1; + } else { + // --inspect + inspectFlag = argv; + + // is number? + if (String(execArgv[inspectFlagIndex + 1]).match(/^[0-9]+$/)) { + // --inspect port + inspectPortCounter = Number.parseInt(execArgv[inspectFlagIndex + 1], 10) + 1; + execArgv.slice(inspectFlagIndex + 1, 1); + } + } + + execArgv[inspectFlagIndex] = inspectFlag + '=' + inspectPortCounter; + } + + return execArgv; + }; + var preserveSymlinksOption = '--preserve-symlinks'; var preserveSymlinksMainOption = '--preserve-symlinks-main'; var nodeOptions = (process && process.env && process.env.NODE_OPTIONS) || []; - var nodeExecArgv = (process && process.execArgv) || []; + var nodeExecArgv = calculateInspectPortOnExecArgv((process && process.execArgv) || []); var isPreserveSymlinksPresent = nodeOptions.includes(preserveSymlinksOption) || nodeExecArgv.includes(preserveSymlinksOption); diff --git a/test/functional/apps/discover/_discover.ts b/test/functional/apps/discover/_discover.ts index aeb02e5c30eb8..def175474d40e 100644 --- a/test/functional/apps/discover/_discover.ts +++ b/test/functional/apps/discover/_discover.ts @@ -105,21 +105,27 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('should modify the time range when the histogram is brushed', async function () { // this is the number of renderings of the histogram needed when new data is fetched // this needs to be improved - const renderingCountInc = 3; + const renderingCountInc = 1; const prevRenderingCount = await elasticChart.getVisualizationRenderingCount(); await PageObjects.timePicker.setDefaultAbsoluteRange(); await PageObjects.discover.waitUntilSearchingHasFinished(); await retry.waitFor('chart rendering complete', async () => { - const actualRenderingCount = await elasticChart.getVisualizationRenderingCount(); - log.debug(`Number of renderings before brushing: ${actualRenderingCount}`); - return actualRenderingCount === prevRenderingCount + renderingCountInc; + const actualCount = await elasticChart.getVisualizationRenderingCount(); + const expectedCount = prevRenderingCount + renderingCountInc; + log.debug( + `renderings before brushing - actual: ${actualCount} expected: ${expectedCount}` + ); + return actualCount === expectedCount; }); await PageObjects.discover.brushHistogram(); await PageObjects.discover.waitUntilSearchingHasFinished(); await retry.waitFor('chart rendering complete after being brushed', async () => { - const actualRenderingCount = await elasticChart.getVisualizationRenderingCount(); - log.debug(`Number of renderings after brushing: ${actualRenderingCount}`); - return actualRenderingCount === prevRenderingCount + 6; + const actualCount = await elasticChart.getVisualizationRenderingCount(); + const expectedCount = prevRenderingCount + renderingCountInc * 2; + log.debug( + `renderings after brushing - actual: ${actualCount} expected: ${expectedCount}` + ); + return actualCount === expectedCount; }); const newDurationHours = await PageObjects.timePicker.getTimeDurationInHours(); expect(Math.round(newDurationHours)).to.be(26); diff --git a/test/functional/apps/discover/_doc_table.ts b/test/functional/apps/discover/_doc_table.ts index 3febeb06fd600..edcb002000183 100644 --- a/test/functional/apps/discover/_doc_table.ts +++ b/test/functional/apps/discover/_doc_table.ts @@ -65,6 +65,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const finalRows = await PageObjects.discover.getDocTableRows(); expect(finalRows.length).to.be.above(initialRows.length); expect(finalRows.length).to.be(rowsHardLimit); + await PageObjects.discover.backToTop(); }); it('should go the end of the table when using the accessible Skip button', async function () { @@ -74,6 +75,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const footer = await PageObjects.discover.getDocTableFooter(); log.debug(await footer.getVisibleText()); expect(await footer.getVisibleText()).to.have.string(rowsHardLimit); + await PageObjects.discover.backToTop(); }); describe('expand a document row', function () { diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index 733f5cb59fbbb..32288239f9848 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -210,6 +210,15 @@ export function DiscoverPageProvider({ getService, getPageObjects }: FtrProvider return skipButton.click(); } + /** + * When scrolling down the legacy table there's a link to scroll up + * So this is done by this function + */ + public async backToTop() { + const skipButton = await testSubjects.find('discoverBackToTop'); + return skipButton.click(); + } + public async getDocTableFooter() { return await testSubjects.find('discoverDocTableFooter'); } diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts index 76f5cb49c7f07..d18e7e427eeca 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts @@ -1820,3 +1820,30 @@ describe('#closePointInTime', () => { expect(mockBaseClient.closePointInTime).toHaveBeenCalledTimes(1); }); }); + +describe('#createPointInTimeFinder', () => { + it('redirects request to underlying base client with default dependencies', () => { + const options = { type: ['a', 'b'], search: 'query' }; + wrapper.createPointInTimeFinder(options); + + expect(mockBaseClient.createPointInTimeFinder).toHaveBeenCalledTimes(1); + expect(mockBaseClient.createPointInTimeFinder).toHaveBeenCalledWith(options, { + client: wrapper, + }); + }); + + it('redirects request to underlying base client with custom dependencies', () => { + const options = { type: ['a', 'b'], search: 'query' }; + const dependencies = { + client: { + find: jest.fn(), + openPointInTimeForType: jest.fn(), + closePointInTime: jest.fn(), + }, + }; + wrapper.createPointInTimeFinder(options, dependencies); + + expect(mockBaseClient.createPointInTimeFinder).toHaveBeenCalledTimes(1); + expect(mockBaseClient.createPointInTimeFinder).toHaveBeenCalledWith(options, dependencies); + }); +}); diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts index 6b06f7e4e68e9..88a89af6be3d0 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts @@ -19,6 +19,8 @@ import type { SavedObjectsClientContract, SavedObjectsClosePointInTimeOptions, SavedObjectsCreateOptions, + SavedObjectsCreatePointInTimeFinderDependencies, + SavedObjectsCreatePointInTimeFinderOptions, SavedObjectsDeleteFromNamespacesOptions, SavedObjectsFindOptions, SavedObjectsFindResponse, @@ -263,6 +265,17 @@ export class EncryptedSavedObjectsClientWrapper implements SavedObjectsClientCon return await this.options.baseClient.closePointInTime(id, options); } + public createPointInTimeFinder( + findOptions: SavedObjectsCreatePointInTimeFinderOptions, + dependencies?: SavedObjectsCreatePointInTimeFinderDependencies + ) { + return this.options.baseClient.createPointInTimeFinder(findOptions, { + client: this, + // Include dependencies last so that subsequent SO client wrappers have their settings applied. + ...dependencies, + }); + } + /** * Strips encrypted attributes from any non-bulk Saved Objects API response. If type isn't * registered, response is returned as is. diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts index 803b36e520a2f..554244dc98be9 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts @@ -1058,6 +1058,36 @@ describe('#closePointInTime', () => { }); }); +describe('#createPointInTimeFinder', () => { + it('redirects request to underlying base client with default dependencies', () => { + const options = { type: ['a', 'b'], search: 'query' }; + client.createPointInTimeFinder(options); + + expect(clientOpts.baseClient.createPointInTimeFinder).toHaveBeenCalledTimes(1); + expect(clientOpts.baseClient.createPointInTimeFinder).toHaveBeenCalledWith(options, { + client, + }); + }); + + it('redirects request to underlying base client with custom dependencies', () => { + const options = { type: ['a', 'b'], search: 'query' }; + const dependencies = { + client: { + find: jest.fn(), + openPointInTimeForType: jest.fn(), + closePointInTime: jest.fn(), + }, + }; + client.createPointInTimeFinder(options, dependencies); + + expect(clientOpts.baseClient.createPointInTimeFinder).toHaveBeenCalledTimes(1); + expect(clientOpts.baseClient.createPointInTimeFinder).toHaveBeenCalledWith( + options, + dependencies + ); + }); +}); + describe('#resolve', () => { const type = 'foo'; const id = `${type}-id`; diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts index 1858bc7108dc9..8378cc4d848cf 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts @@ -16,6 +16,8 @@ import type { SavedObjectsClientContract, SavedObjectsClosePointInTimeOptions, SavedObjectsCreateOptions, + SavedObjectsCreatePointInTimeFinderDependencies, + SavedObjectsCreatePointInTimeFinderOptions, SavedObjectsDeleteFromNamespacesOptions, SavedObjectsFindOptions, SavedObjectsOpenPointInTimeOptions, @@ -616,6 +618,20 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra return await this.baseClient.closePointInTime(id, options); } + public createPointInTimeFinder( + findOptions: SavedObjectsCreatePointInTimeFinderOptions, + dependencies?: SavedObjectsCreatePointInTimeFinderDependencies + ) { + // We don't need to perform an authorization check here or add an audit log, because + // `createPointInTimeFinder` is simply a helper that calls `find`, `openPointInTimeForType`, + // and `closePointInTime` internally, so authz checks and audit logs will already be applied. + return this.baseClient.createPointInTimeFinder(findOptions, { + client: this, + // Include dependencies last so that subsequent SO client wrappers have their settings applied. + ...dependencies, + }); + } + private async checkPrivileges( actions: string | string[], namespaceOrNamespaces?: string | Array diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts index fa53f110e30c3..cbb71d4bbcf81 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts @@ -643,5 +643,43 @@ const ERROR_NAMESPACE_SPECIFIED = 'Spaces currently determines the namespaces'; }); }); }); + + describe('#createPointInTimeFinder', () => { + test(`throws error if options.namespace is specified`, async () => { + const { client } = createSpacesSavedObjectsClient(); + + const options = { type: ['a', 'b'], search: 'query', namespace: 'oops' }; + expect(() => client.createPointInTimeFinder(options)).toThrow(ERROR_NAMESPACE_SPECIFIED); + }); + + it('redirects request to underlying base client with default dependencies', () => { + const { client, baseClient } = createSpacesSavedObjectsClient(); + + const options = { type: ['a', 'b'], search: 'query' }; + client.createPointInTimeFinder(options); + + expect(baseClient.createPointInTimeFinder).toHaveBeenCalledTimes(1); + expect(baseClient.createPointInTimeFinder).toHaveBeenCalledWith(options, { + client, + }); + }); + + it('redirects request to underlying base client with custom dependencies', () => { + const { client, baseClient } = createSpacesSavedObjectsClient(); + + const options = { type: ['a', 'b'], search: 'query' }; + const dependencies = { + client: { + find: jest.fn(), + openPointInTimeForType: jest.fn(), + closePointInTime: jest.fn(), + }, + }; + client.createPointInTimeFinder(options, dependencies); + + expect(baseClient.createPointInTimeFinder).toHaveBeenCalledTimes(1); + expect(baseClient.createPointInTimeFinder).toHaveBeenCalledWith(options, dependencies); + }); + }); }); }); diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index f70714b8ad102..c544e2f46f058 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -18,6 +18,8 @@ import type { SavedObjectsClientContract, SavedObjectsClosePointInTimeOptions, SavedObjectsCreateOptions, + SavedObjectsCreatePointInTimeFinderDependencies, + SavedObjectsCreatePointInTimeFinderOptions, SavedObjectsDeleteFromNamespacesOptions, SavedObjectsFindOptions, SavedObjectsOpenPointInTimeOptions, @@ -420,4 +422,31 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { namespace: spaceIdToNamespace(this.spaceId), }); } + + /** + * Returns a generator to help page through large sets of saved objects. + * + * The generator wraps calls to `SavedObjects.find` and iterates over + * multiple pages of results using `_pit` and `search_after`. This will + * open a new Point In Time (PIT), and continue paging until a set of + * results is received that's smaller than the designated `perPage`. + * + * @param {object} findOptions - {@link SavedObjectsCreatePointInTimeFinderOptions} + * @param {object} [dependencies] - {@link SavedObjectsCreatePointInTimeFinderDependencies} + */ + createPointInTimeFinder( + findOptions: SavedObjectsCreatePointInTimeFinderOptions, + dependencies?: SavedObjectsCreatePointInTimeFinderDependencies + ) { + throwErrorIfNamespaceSpecified(findOptions); + // We don't need to handle namespaces here, because `createPointInTimeFinder` + // is simply a helper that calls `find`, `openPointInTimeForType`, and + // `closePointInTime` internally, so namespaces will already be handled + // in those methods. + return this.client.createPointInTimeFinder(findOptions, { + client: this, + // Include dependencies last so that subsequent SO client wrappers have their settings applied. + ...dependencies, + }); + } }