prerequesites part 1 for lens ESQL generation (elastic#203962)

packages/kbn-es-query/src/filters/build_filters/exists_filter.ts

@@ -57,3 +57,16 @@ export const buildExistsFilter = (field: DataViewFieldBase, indexPattern: DataVi
     },
   } as ExistsFilter;
 };
+
+export const buildSimpleExistFilter = (fieldName: string, dataViewId: string) => {
+  return {
+    meta: {
+      index: dataViewId,
+    },
+    query: {
+      exists: {
+        field: fieldName,
+      },
+    },
+  } as ExistsFilter;
+};

packages/kbn-es-query/src/filters/build_filters/range_filter.ts

@@ -187,6 +187,20 @@ export const buildRangeFilter = (
   }
 };
 
+export const buildSimpleNumberRangeFilter = (
+  fieldName: string,
+  params: RangeFilterParams,
+  value: string,
+  dataViewId: string
+) => {
+  return buildRangeFilter(
+    { name: fieldName, type: 'number' },
+    params,
+    { id: dataViewId, title: dataViewId },
+    value
+  );
+};
+
 /**
  * @internal
  */

src/platform/packages/shared/kbn-esql-utils/index.ts

@@ -33,6 +33,7 @@ export {
   isESQLColumnGroupable,
   isESQLFieldGroupable,
   TextBasedLanguages,
+  sanitazeESQLInput,
   queryCannotBeSampled,
 } from './src';
 

src/platform/packages/shared/kbn-esql-utils/src/index.ts

@@ -37,3 +37,4 @@ export {
   isESQLColumnGroupable,
   isESQLFieldGroupable,
 } from './utils/esql_fields_utils';
+export { sanitazeESQLInput } from './utils/sanitaze_input';

src/platform/packages/shared/kbn-esql-utils/src/utils/append_to_query.ts

@@ -8,6 +8,7 @@
  */
 
 import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+import { sanitazeESQLInput } from './sanitaze_input';
 
 // Append in a new line the appended text to take care of the case where the user adds a comment at the end of the query
 // in these cases a base query such as "from index // comment" will result in errors or wrong data if we don't append in a new line
@@ -43,7 +44,7 @@ export function appendWhereClauseToESQLQuery(
   let filterValue =
     typeof value === 'string' ? `"${value.replace(/\\/g, '\\\\').replace(/\"/g, '\\"')}"` : value;
   // Adding the backticks here are they are needed for special char fields
-  let fieldName = `\`${field}\``;
+  let fieldName = sanitazeESQLInput(field);
 
   // casting to string
   // there are some field types such as the ip that need

src/platform/packages/shared/kbn-esql-utils/src/utils/sanitaze_input.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export function sanitazeESQLInput(input: string): string | undefined {
+  return `\`${input.replace(/`/g, '``')}\``;
+}

src/plugins/data/common/search/aggs/buckets/lib/time_buckets/time_buckets.ts

@@ -23,7 +23,7 @@ import {
 } from './calc_es_interval';
 import { autoInterval } from '../../_interval_options';
 
-interface TimeBucketsInterval extends moment.Duration {
+export interface TimeBucketsInterval extends moment.Duration {
   // TODO double-check whether all of these are needed
   description: string;
   esValue: EsInterval['value'];

src/plugins/data/common/search/aggs/utils/calculate_auto_time_expression.ts

@@ -8,14 +8,36 @@
  */
 
 import moment from 'moment';
+import { TimeBucketsInterval } from '../buckets/lib/time_buckets/time_buckets';
 import { UI_SETTINGS } from '../../../constants';
 import { TimeRange } from '../../../query';
 import { TimeBuckets } from '../buckets/lib/time_buckets';
 import { toAbsoluteDates } from './date_interval_utils';
 import { autoInterval } from '../buckets/_interval_options';
 
 export function getCalculateAutoTimeExpression(getConfig: (key: string) => any) {
-  return function calculateAutoTimeExpression(range: TimeRange) {
+  function calculateAutoTimeExpression(range: TimeRange): string | undefined;
+  function calculateAutoTimeExpression(
+    range: TimeRange,
+    interval: string,
+    asExpression?: true
+  ): string | undefined;
+  function calculateAutoTimeExpression(
+    range: TimeRange,
+    interval: string,
+    asExpression: false
+  ): TimeBucketsInterval | undefined;
+  function calculateAutoTimeExpression(
+    range: TimeRange,
+    interval?: string,
+    asExpression?: boolean
+  ): string | TimeBucketsInterval | undefined;
+
+  function calculateAutoTimeExpression(
+    range: TimeRange,
+    interval: string = autoInterval,
+    asExpression: boolean = true
+  ): string | TimeBucketsInterval | undefined {
     const dates = toAbsoluteDates(range);
     if (!dates) {
       return;
@@ -28,12 +50,18 @@ export function getCalculateAutoTimeExpression(getConfig: (key: string) => any)
       'dateFormat:scaled': getConfig('dateFormat:scaled'),
     });
 
-    buckets.setInterval(autoInterval);
+    buckets.setInterval(interval);
     buckets.setBounds({
       min: moment(dates.from),
       max: moment(dates.to),
     });
 
-    return buckets.getInterval().expression;
-  };
+    const intervalResult = buckets.getInterval();
+    if (asExpression) {
+      return intervalResult.expression;
+    }
+    return intervalResult;
+  }
+
+  return calculateAutoTimeExpression;
 }

src/plugins/data/common/search/expressions/esql.ts

@@ -17,7 +17,7 @@ import type {
 } from '@kbn/search-types';
 import type { Datatable, ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common';
 import { RequestAdapter } from '@kbn/inspector-plugin/common';
-import { getStartEndParams } from '@kbn/esql-utils';
+import { getIndexPatternFromESQLQuery, getStartEndParams } from '@kbn/esql-utils';
 import { zipObject } from 'lodash';
 import { catchError, defer, map, Observable, switchMap, tap, throwError } from 'rxjs';
 import { buildEsQuery, type Filter } from '@kbn/es-query';
@@ -58,6 +58,7 @@ interface Arguments {
    */
   titleForInspector?: string;
   descriptionForInspector?: string;
+  ignoreGlobalFilters?: boolean;
 }
 
 export type EsqlExpressionFunctionDefinition = ExpressionFunctionDefinition<
@@ -140,10 +141,24 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
           defaultMessage: 'The description to show in Inspector.',
         }),
       },
+      ignoreGlobalFilters: {
+        types: ['boolean'],
+        default: false,
+        help: i18n.translate('data.search.esql.ignoreGlobalFilters.help', {
+          defaultMessage: 'Whether to ignore or use global query and filters',
+        }),
+      },
     },
     fn(
       input,
-      { query, /* timezone, */ timeField, locale, titleForInspector, descriptionForInspector },
+      {
+        query,
+        /* timezone, */ timeField,
+        locale,
+        titleForInspector,
+        descriptionForInspector,
+        ignoreGlobalFilters,
+      },
       { abortSignal, inspectorAdapters, getKibanaRequest }
     ) {
       return defer(() =>
@@ -202,7 +217,7 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
               : undefined;
 
             const filters = [
-              ...(input.filters ?? []),
+              ...(ignoreGlobalFilters ? [] : input.filters ?? []),
               ...(timeFilter ? [timeFilter] : []),
               ...(delayFilter ? [delayFilter] : []),
             ];
@@ -311,6 +326,8 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
           const lookup = new Set(
             hasEmptyColumns ? body.columns?.map(({ name }) => name) || [] : []
           );
+          const indexPattern = getIndexPatternFromESQLQuery(query);
+
           const allColumns =
             (body.all_columns ?? body.columns)?.map(({ name, type }) => ({
               id: name,
@@ -323,8 +340,11 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
                     ? {
                         appliedTimeRange: input?.timeRange,
                         params: {},
+                        indexPattern,
                       }
-                    : {},
+                    : {
+                        indexPattern,
+                      },
               },
               isNull: hasEmptyColumns ? !lookup.has(name) : false,
             })) ?? [];
@@ -341,6 +361,10 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
             type: 'datatable',
             meta: {
               type: ESQL_TABLE_TYPE,
+              query,
+              statistics: {
+                totalCount: body.values.length,
+              },
             },
             columns: allColumns,
             rows,