[ML] Log rate spike: search (#136571)

* add search bar with filters

* create application directory and utils directory

* incorporate cache fix and ensure search works

* move search above table and fix type errors

* refactor to minimize start bundle

* update branch with latest changes and fix types

* add comments to track duplicate code

x-pack/plugins/aiops/kibana.json

@@ -12,7 +12,9 @@
   "requiredPlugins": [
     "charts",
     "data",
-    "licensing"
+    "discover",
+    "licensing",
+    "unifiedSearch"
   ],
   "optionalPlugins": [],
   "requiredBundles": ["kibanaReact", "fieldFormats"],

x-pack/plugins/aiops/public/application/services/time_field_range.ts

@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+// TODO Consolidate with near duplicate service in
+// `x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_field_range.ts`
+
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import { getCoreStart } from '../../kibana_services';

x-pack/plugins/aiops/public/application/services/timefilter_refresh_service.ts

@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+// TODO Consolidate with near duplicate service in
+// `x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/timefilter_refresh_service.ts`
+
 import { Subject } from 'rxjs';
 
 export interface Refresh {

x-pack/plugins/aiops/common/error_utils.ts → x-pack/plugins/aiops/public/application/utils/error_utils.ts

@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+// TODO Consolidate with duplicate error utils file in
+// `x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/error_utils.ts`
+
 import type { IHttpFetchError } from '@kbn/core-http-browser';
 import Boom from '@hapi/boom';
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';

x-pack/plugins/aiops/public/query_utils.ts → x-pack/plugins/aiops/public/application/utils/query_utils.ts

@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+// TODO Consolidate with duplicate query utils in
+// `x-pack/plugins/data_visualizer/common/utils/query_utils.ts`
+
 import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { Query } from '@kbn/es-query';

x-pack/plugins/aiops/public/application/utils/search_utils.ts

@@ -0,0 +1,231 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// TODO Consolidate with duplicate saved search utils file in
+// `x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts`
+
+import { cloneDeep } from 'lodash';
+import { IUiSettingsClient } from '@kbn/core/public';
+import { SearchSource } from '@kbn/data-plugin/common';
+import { SavedSearch } from '@kbn/discover-plugin/public';
+import { FilterManager } from '@kbn/data-plugin/public';
+import {
+  fromKueryExpression,
+  toElasticsearchQuery,
+  buildQueryFromFilters,
+  buildEsQuery,
+  Query,
+  Filter,
+} from '@kbn/es-query';
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import { getEsQueryConfig } from '@kbn/data-plugin/common';
+import type { SimpleSavedObject } from '@kbn/core/public';
+import { isPopulatedObject } from '@kbn/ml-is-populated-object';
+
+const DEFAULT_QUERY = {
+  bool: {
+    must: [
+      {
+        match_all: {},
+      },
+    ],
+  },
+};
+
+export const SEARCH_QUERY_LANGUAGE = {
+  KUERY: 'kuery',
+  LUCENE: 'lucene',
+} as const;
+
+export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE];
+
+export function getDefaultQuery() {
+  return cloneDeep(DEFAULT_QUERY);
+}
+
+export type SavedSearchSavedObject = SimpleSavedObject<any>;
+
+export function isSavedSearchSavedObject(arg: unknown): arg is SavedSearchSavedObject {
+  return isPopulatedObject(arg, ['id', 'type', 'attributes']);
+}
+
+/**
+ * Parse the stringified searchSourceJSON
+ * from a saved search or saved search object
+ */
+export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObject | SavedSearch) {
+  const search = isSavedSearchSavedObject(savedSearch)
+    ? savedSearch?.attributes?.kibanaSavedObjectMeta
+    : // @ts-ignore
+      savedSearch?.kibanaSavedObjectMeta;
+
+  const parsed =
+    typeof search?.searchSourceJSON === 'string'
+      ? (JSON.parse(search.searchSourceJSON) as {
+          query: Query;
+          filter: Filter[];
+        })
+      : undefined;
+
+  // Remove indexRefName because saved search might no longer be relevant
+  // if user modifies the query or filter
+  // after opening a saved search
+  if (parsed && Array.isArray(parsed.filter)) {
+    parsed.filter.forEach((f) => {
+      // @ts-expect-error indexRefName does appear in meta for newly created saved search
+      f.meta.indexRefName = undefined;
+    });
+  }
+  return parsed;
+}
+
+/**
+ * Create an Elasticsearch query that combines both lucene/kql query string and filters
+ * Should also form a valid query if only the query or filters is provided
+ */
+export function createMergedEsQuery(
+  query?: Query,
+  filters?: Filter[],
+  dataView?: DataView,
+  uiSettings?: IUiSettingsClient
+) {
+  let combinedQuery: QueryDslQueryContainer = getDefaultQuery();
+
+  if (query && query.language === SEARCH_QUERY_LANGUAGE.KUERY) {
+    const ast = fromKueryExpression(query.query);
+    if (query.query !== '') {
+      combinedQuery = toElasticsearchQuery(ast, dataView);
+    }
+    if (combinedQuery.bool !== undefined) {
+      const filterQuery = buildQueryFromFilters(filters, dataView);
+
+      if (!Array.isArray(combinedQuery.bool.filter)) {
+        combinedQuery.bool.filter =
+          combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter];
+      }
+
+      if (!Array.isArray(combinedQuery.bool.must_not)) {
+        combinedQuery.bool.must_not =
+          combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not];
+      }
+
+      combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter];
+      combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not];
+    }
+  } else {
+    combinedQuery = buildEsQuery(
+      dataView,
+      query ? [query] : [],
+      filters ? filters : [],
+      uiSettings ? getEsQueryConfig(uiSettings) : undefined
+    );
+  }
+  return combinedQuery;
+}
+
+/**
+ * Extract query data from the saved search object
+ * with overrides from the provided query data and/or filters
+ */
+export function getEsQueryFromSavedSearch({
+  dataView,
+  uiSettings,
+  savedSearch,
+  query,
+  filters,
+  filterManager,
+}: {
+  dataView: DataView;
+  uiSettings: IUiSettingsClient;
+  savedSearch: SavedSearchSavedObject | SavedSearch | null | undefined;
+  query?: Query;
+  filters?: Filter[];
+  filterManager?: FilterManager;
+}) {
+  if (!dataView || !savedSearch) return;
+
+  const userQuery = query;
+  const userFilters = filters;
+
+  // If saved search has a search source with nested parent
+  // e.g. a search coming from Dashboard saved search embeddable
+  // which already combines both the saved search's original query/filters and the Dashboard's
+  // then no need to process any further
+  if (
+    savedSearch &&
+    'searchSource' in savedSearch &&
+    savedSearch?.searchSource instanceof SearchSource &&
+    savedSearch.searchSource.getParent() !== undefined &&
+    userQuery
+  ) {
+    // Flattened query from search source may contain a clause that narrows the time range
+    // which might interfere with global time pickers so we need to remove
+    const savedQuery =
+      cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultQuery();
+    const timeField = savedSearch.searchSource.getField('index')?.timeFieldName;
+
+    if (Array.isArray(savedQuery.bool.filter) && timeField !== undefined) {
+      savedQuery.bool.filter = savedQuery.bool.filter.filter(
+        (c: QueryDslQueryContainer) =>
+          !(c.hasOwnProperty('range') && c.range?.hasOwnProperty(timeField))
+      );
+    }
+    return {
+      searchQuery: savedQuery,
+      searchString: userQuery.query,
+      queryLanguage: userQuery.language as SearchQueryLanguage,
+    };
+  }
+
+  // TODO: support saved search
+  // If saved search is an json object with the original query and filter
+  // retrieve the parsed query and filter
+  const savedSearchData = undefined; // getQueryFromSavedSearchObject(savedSearch);
+
+  // If no saved search available, use user's query and filters
+  if (!savedSearchData && userQuery) {
+    if (filterManager && userFilters) filterManager.setFilters(userFilters);
+
+    const combinedQuery = createMergedEsQuery(
+      userQuery,
+      Array.isArray(userFilters) ? userFilters : [],
+      dataView,
+      uiSettings
+    );
+
+    return {
+      searchQuery: combinedQuery,
+      searchString: userQuery.query,
+      queryLanguage: userQuery.language as SearchQueryLanguage,
+    };
+  }
+
+  // If saved search available, merge saved search with latest user query or filters
+  // which might differ from extracted saved search data
+  if (savedSearchData) {
+    // @ts-ignore property does not exist on type never
+    const currentQuery = userQuery ?? savedSearchData?.query;
+    // @ts-ignore property does not exist on type never
+    const currentFilters = userFilters ?? savedSearchData?.filter;
+
+    if (filterManager) filterManager.setFilters(currentFilters);
+
+    const combinedQuery = createMergedEsQuery(
+      currentQuery,
+      Array.isArray(currentFilters) ? currentFilters : [],
+      dataView,
+      uiSettings
+    );
+
+    return {
+      searchQuery: combinedQuery,
+      searchString: currentQuery.query,
+      queryLanguage: currentQuery.language as SearchQueryLanguage,
+    };
+  }
+}

x-pack/plugins/aiops/public/components/date_picker_wrapper/date_picker_wrapper.tsx

@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+// TODO Consolidate with duplicate component `DatePickerWrapper` in
+// `x-pack/plugins/data_visualizer/public/application/common/components/date_picker_wrapper/date_picker_wrapper.tsx`
+
 import React, { FC, useCallback, useEffect, useMemo, useState } from 'react';
 import { Subscription } from 'rxjs';
 import { debounce } from 'lodash';

x-pack/plugins/aiops/public/components/document_count_content/total_count_header/total_count_header.tsx

@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+// TODO Consolidate with duplicate component `TotalCountHeader` in
+// `x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/total_count_header.tsx`
+
 import { EuiFlexItem, EuiText } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n-react';
 import React from 'react';