Search Sessions Stabilization Stage I (elastic#134983)

Changes search service/search session infrastructure to improve performance, stability, and resiliency by ensuring that search sessions don’t add additional load on a cluster when the feature is not used

docs/settings/search-sessions-settings.asciidoc

@@ -10,9 +10,6 @@ Configure the search session settings in your `kibana.yml` configuration file.
 `data.search.sessions.enabled` {ess-icon}::
 Set to `true` (default) to enable search sessions.
 
-`data.search.sessions.trackingInterval` {ess-icon}::
-The frequency for updating the state of a search session. The default is `10s`.
-
 `data.search.sessions.pageSize` {ess-icon}::
 How many search sessions {kib} processes at once while monitoring
 session progress. The default is `100`.
@@ -21,9 +18,6 @@ session progress. The default is `100`.
 How long {kib} stores search results from unsaved sessions,
 after the last search in the session completes. The default is `5m`.
 
-`data.search.sessions.notTouchedInProgressTimeout` {ess-icon}::
-How long a search session can run after a user navigates away without saving a session. The default is `1m`.
-
 `data.search.sessions.maxUpdateRetries` {ess-icon}::
 How many retries {kib} can perform while attempting to save a search session. The default is `3`.
 

src/plugins/data/common/search/session/status.ts

@@ -13,3 +13,9 @@ export enum SearchSessionStatus {
   CANCELLED = 'cancelled',
   EXPIRED = 'expired',
 }
+
+export enum SearchStatus {
+  IN_PROGRESS = 'in_progress',
+  ERROR = 'error',
+  COMPLETE = 'complete',
+}

src/plugins/data/common/search/session/types.ts

@@ -6,8 +6,9 @@
  * Side Public License, v 1.
  */
 
+import type { SavedObjectsFindResponse } from '@kbn/core/server';
 import { SerializableRecord } from '@kbn/utility-types';
-import { SearchSessionStatus } from './status';
+import type { SearchSessionStatus, SearchStatus } from './status';
 
 export const SEARCH_SESSION_TYPE = 'search-session';
 export interface SearchSessionSavedObjectAttributes {
@@ -24,25 +25,12 @@ export interface SearchSessionSavedObjectAttributes {
    * Creation time of the session
    */
   created: string;
-  /**
-   * Last touch time of the session
-   */
-  touched: string;
+
   /**
    * Expiration time of the session. Expiration itself is managed by Elasticsearch.
    */
   expires: string;
-  /**
-   * Time of transition into completed state,
-   *
-   * Can be "null" in case already completed session
-   * transitioned into in-progress session
-   */
-  completed?: string | null;
-  /**
-   * status
-   */
-  status: SearchSessionStatus;
+
   /**
    * locatorId (see share.url.locators service)
    */
@@ -62,10 +50,6 @@ export interface SearchSessionSavedObjectAttributes {
    */
   idMapping: Record<string, SearchSessionRequestInfo>;
 
-  /**
-   * This value is true if the session was actively stored by the user. If it is false, the session may be purged by the system.
-   */
-  persisted: boolean;
   /**
    * The realm type/name & username uniquely identifies the user who created this search session
    */
@@ -76,6 +60,11 @@ export interface SearchSessionSavedObjectAttributes {
    * Version information to display warnings when trying to restore a session from a different version
    */
   version: string;
+
+  /**
+   * `true` if session was cancelled
+   */
+  isCanceled?: boolean;
 }
 
 export interface SearchSessionRequestInfo {
@@ -87,20 +76,30 @@ export interface SearchSessionRequestInfo {
    * Search strategy used to submit the search request
    */
   strategy: string;
-  /**
-   * status
-   */
-  status: string;
+}
+
+export interface SearchSessionRequestStatus {
+  status: SearchStatus;
   /**
    * An optional error. Set if status is set to error.
    */
   error?: string;
 }
 
-export interface SearchSessionFindOptions {
-  page?: number;
-  perPage?: number;
-  sortField?: string;
-  sortOrder?: string;
-  filter?: string;
+/**
+ * On-the-fly calculated search session status
+ */
+export interface SearchSessionStatusResponse {
+  status: SearchSessionStatus;
+}
+
+/**
+ * List of search session objects with on-the-fly calculated search session statuses
+ */
+export interface SearchSessionsFindResponse
+  extends SavedObjectsFindResponse<SearchSessionSavedObjectAttributes> {
+  /**
+   * Map containing calculated statuses of search sessions from the find response
+   */
+  statuses: Record<string, SearchSessionStatusResponse>;
 }

src/plugins/data/common/search/types.ts

@@ -72,6 +72,11 @@ export interface IKibanaSearchResponse<RawResponse = any> {
    */
   isRestored?: boolean;
 
+  /**
+   * Indicates whether the search has been saved to a search-session object and long keepAlive was set
+   */
+  isStored?: boolean;
+
   /**
    * Optional warnings returned from Elasticsearch (for example, deprecation warnings)
    */
@@ -119,6 +124,11 @@ export interface ISearchOptions {
    */
   isStored?: boolean;
 
+  /**
+   * Whether the search was successfully polled after session was saved. Search was added to a session saved object and keepAlive extended.
+   */
+  isSearchStored?: boolean;
+
   /**
    * Whether the session is restored (i.e. search requests should re-use the stored search IDs,
    * rather than starting from scratch)
@@ -148,5 +158,11 @@ export interface ISearchOptions {
  */
 export type ISearchOptionsSerializable = Pick<
   ISearchOptions,
-  'strategy' | 'legacyHitsTotal' | 'sessionId' | 'isStored' | 'isRestore' | 'executionContext'
+  | 'strategy'
+  | 'legacyHitsTotal'
+  | 'sessionId'
+  | 'isStored'
+  | 'isSearchStored'
+  | 'isRestore'
+  | 'executionContext'
 >;

src/plugins/data/config.ts

@@ -13,42 +13,13 @@ export const searchSessionsConfigSchema = schema.object({
    * Turns the feature on \ off (incl. removing indicator and management screens)
    */
   enabled: schema.boolean({ defaultValue: true }),
-  /**
-   * pageSize controls how many search session objects we load at once while monitoring
-   * session completion
-   */
-  pageSize: schema.number({ defaultValue: 100 }),
-  /**
-   * trackingInterval controls how often we track persisted search session objects progress
-   */
-  trackingInterval: schema.duration({ defaultValue: '10s' }),
-
-  /**
-   * cleanupInterval controls how often we track non-persisted search session objects for cleanup
-   */
-  cleanupInterval: schema.duration({ defaultValue: '60s' }),
-
-  /**
-   * expireInterval controls how often we track persisted search session objects for expiration
-   */
-  expireInterval: schema.duration({ defaultValue: '60m' }),
-
-  /**
-   * monitoringTaskTimeout controls for how long task manager waits for search session monitoring task to complete before considering it timed out,
-   * If tasks timeouts it receives cancel signal and next task starts in "trackingInterval" time
-   */
-  monitoringTaskTimeout: schema.duration({ defaultValue: '5m' }),
 
   /**
-   * notTouchedTimeout controls how long do we store unpersisted search session results,
-   * after the last search in the session has completed
+   * notTouchedTimeout controls how long user can save a session after all searches completed.
+   * The client continues to poll searches to keep the alive until this timeout hits
    */
   notTouchedTimeout: schema.duration({ defaultValue: '5m' }),
-  /**
-   * notTouchedInProgressTimeout controls how long do allow a search session to run after
-   * a user has navigated away without persisting
-   */
-  notTouchedInProgressTimeout: schema.duration({ defaultValue: '1m' }),
+
   /**
    * maxUpdateRetries controls how many retries we perform while attempting to save a search session
    */
@@ -60,15 +31,15 @@ export const searchSessionsConfigSchema = schema.object({
   defaultExpiration: schema.duration({ defaultValue: '7d' }),
   management: schema.object({
     /**
-     * maxSessions controls how many saved search sessions we display per page on the management screen.
+     * maxSessions controls how many saved search sessions we load on the management screen.
      */
-    maxSessions: schema.number({ defaultValue: 10000 }),
+    maxSessions: schema.number({ defaultValue: 100 }),
     /**
-     * refreshInterval controls how often we refresh the management screen.
+     * refreshInterval controls how often we refresh the management screen. 0s as duration means that auto-refresh is turned off.
      */
-    refreshInterval: schema.duration({ defaultValue: '10s' }),
+    refreshInterval: schema.duration({ defaultValue: '0s' }),
     /**
-     * refreshTimeout controls how often we refresh the management screen.
+     * refreshTimeout controls the timeout for loading search sessions on mgmt screen
      */
     refreshTimeout: schema.duration({ defaultValue: '1m' }),
     expiresSoonWarning: schema.duration({ defaultValue: '1d' }),