[ES|QL] Editor supports integrations (elastic#184716)

## Summary

Supporting integrations in the editor using the native monaco
suggestions system.


![meow](https://github.com/elastic/kibana/assets/17003240/0e72ebd0-c71e-4282-b444-6106feb3926c)

This PR:
- Uses a different icon for the simple indices, I didn't like the
previous one
- Adds support of integrations using the built in monaco api
capabilities
- The integrations have a different icon and a different indicator
(Integrations vs Indices)
- I am fetching the integrations from the api that was given to me. The
api is hardcoded because is very difficult to get it from the fleet
plugin, I asked from the team to move their constants in a package
elastic#186061


### Checklist

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Kibana Machine <[email protected]>

packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts

@@ -21,6 +21,16 @@ import { getParamAtPosition } from './helper';
 import { nonNullable } from '../shared/helpers';
 import { METADATA_FIELDS } from '../shared/constants';
 
+interface Integration {
+  name: string;
+  hidden: boolean;
+  title?: string;
+  dataStreams: Array<{
+    name: string;
+    title?: string;
+  }>;
+}
+
 const triggerCharacters = [',', '(', '=', ' '];
 
 const fields: Array<{ name: string; type: string; suggestedAs?: string }> = [
@@ -42,20 +52,29 @@ const fields: Array<{ name: string; type: string; suggestedAs?: string }> = [
   { name: 'kubernetes.something.something', type: 'number' },
 ];
 
-const indexes = (
-  [] as Array<{ name: string; hidden: boolean; suggestedAs: string | undefined }>
-).concat(
+const indexes = ([] as Array<{ name: string; hidden: boolean; suggestedAs?: string }>).concat(
   ['a', 'index', 'otherIndex', '.secretIndex', 'my-index'].map((name) => ({
     name,
     hidden: name.startsWith('.'),
-    suggestedAs: undefined,
   })),
   ['my-index[quoted]', 'my-index$', 'my_index{}'].map((name) => ({
     name,
     hidden: false,
     suggestedAs: `\`${name}\``,
   }))
 );
+
+const integrations: Integration[] = ['nginx', 'k8s'].map((name) => ({
+  name,
+  hidden: false,
+  title: `integration-${name}`,
+  dataStreams: [
+    {
+      name: `${name}-1`,
+      title: `integration-${name}-1`,
+    },
+  ],
+}));
 const policies = [
   {
     name: 'policy',
@@ -347,9 +366,8 @@ describe('autocomplete', () => {
   });
 
   describe('from', () => {
-    const suggestedIndexes = indexes
-      .filter(({ hidden }) => !hidden)
-      .map(({ name, suggestedAs }) => suggestedAs || name);
+    const suggestedIndexes = indexes.filter(({ hidden }) => !hidden).map(({ name }) => name);
+
     // Monaco will filter further down here
     testSuggestions(
       'f',
@@ -372,6 +390,16 @@ describe('autocomplete', () => {
       METADATA_FIELDS.filter((field) => field !== '_index'),
       ' '
     );
+
+    // with integrations support
+    const dataSources = indexes.concat(integrations);
+    const suggestedDataSources = dataSources
+      .filter(({ hidden }) => !hidden)
+      .map(({ name }) => name);
+
+    testSuggestions('from ', suggestedDataSources, '', [undefined, dataSources, undefined]);
+    testSuggestions('from a,', suggestedDataSources, '', [undefined, dataSources, undefined]);
+    testSuggestions('from *,', suggestedDataSources, '', [undefined, dataSources, undefined]);
   });
 
   describe('show', () => {

packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts

@@ -84,11 +84,19 @@ import {
   getFunctionsToIgnoreForStats,
   getParamAtPosition,
   getQueryForFields,
+  getSourcesFromCommands,
   isAggFunctionUsedAlready,
 } from './helper';
 import { FunctionArgSignature } from '../definitions/types';
 
 type GetSourceFn = () => Promise<SuggestionRawDefinition[]>;
+type GetDataSourceFn = (sourceName: string) => Promise<
+  | {
+      name: string;
+      dataStreams?: Array<{ name: string; title?: string }>;
+    }
+  | undefined
+>;
 type GetFieldsByTypeFn = (
   type: string | string[],
   ignored?: string[]
@@ -203,6 +211,7 @@ export async function suggest(
     resourceRetriever
   );
   const getSources = getSourcesRetriever(resourceRetriever);
+  const getDatastreamsForIntegration = getDatastreamsForIntegrationRetriever(resourceRetriever);
   const { getPolicies, getPolicyMetadata } = getPolicyRetriever(resourceRetriever);
 
   if (astContext.type === 'newCommand') {
@@ -223,6 +232,7 @@ export async function suggest(
       ast,
       astContext,
       getSources,
+      getDatastreamsForIntegration,
       getFieldsByType,
       getFieldsMap,
       getPolicies,
@@ -303,7 +313,21 @@ function getSourcesRetriever(resourceRetriever?: ESQLCallbacks) {
   return async () => {
     const list = (await helper()) || [];
     // hide indexes that start with .
-    return buildSourcesDefinitions(list.filter(({ hidden }) => !hidden).map(({ name }) => name));
+    return buildSourcesDefinitions(
+      list
+        .filter(({ hidden }) => !hidden)
+        .map(({ name, dataStreams, title }) => {
+          return { name, isIntegration: Boolean(dataStreams && dataStreams.length), title };
+        })
+    );
+  };
+}
+
+function getDatastreamsForIntegrationRetriever(resourceRetriever?: ESQLCallbacks) {
+  const helper = getSourcesHelper(resourceRetriever);
+  return async (sourceName: string) => {
+    const list = (await helper()) || [];
+    return list.find(({ name }) => name === sourceName);
   };
 }
 
@@ -475,6 +499,7 @@ async function getExpressionSuggestionsByType(
     node: ESQLSingleAstItem | undefined;
   },
   getSources: GetSourceFn,
+  getDatastreamsForIntegration: GetDataSourceFn,
   getFieldsByType: GetFieldsByTypeFn,
   getFieldsMap: GetFieldsMapFn,
   getPolicies: GetPoliciesFn,
@@ -829,9 +854,21 @@ async function getExpressionSuggestionsByType(
         const policies = await getPolicies();
         suggestions.push(...(policies.length ? policies : [buildNoPoliciesAvailableDefinition()]));
       } else {
-        // FROM <suggest>
-        // @TODO: filter down the suggestions here based on other existing sources defined
-        suggestions.push(...(await getSources()));
+        const index = getSourcesFromCommands(commands, 'index');
+        // This is going to be empty for simple indices, and not empty for integrations
+        if (index && index.text) {
+          const source = index.text.replace(EDITOR_MARKER, '');
+          const dataSource = await getDatastreamsForIntegration(source);
+          const newDefinitions = buildSourcesDefinitions(
+            dataSource?.dataStreams?.map(({ name }) => ({ name, isIntegration: false })) || []
+          );
+          suggestions.push(...newDefinitions);
+        } else {
+          // FROM <suggest>
+          // @TODO: filter down the suggestions here based on other existing sources defined
+          const sourcesDefinitions = await getSources();
+          suggestions.push(...sourcesDefinitions);
+        }
       }
     }
   }

packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts

@@ -143,14 +143,22 @@ export const buildVariablesDefinitions = (variables: string[]): SuggestionRawDef
     sortText: 'D',
   }));
 
-export const buildSourcesDefinitions = (sources: string[]): SuggestionRawDefinition[] =>
-  sources.map((label) => ({
-    label,
-    text: getSafeInsertText(label, { dashSupported: true }),
-    kind: 'Reference',
-    detail: i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.sourceDefinition', {
-      defaultMessage: `Index`,
-    }),
+export const buildSourcesDefinitions = (
+  sources: Array<{ name: string; isIntegration: boolean; title?: string }>
+): SuggestionRawDefinition[] =>
+  sources.map(({ name, isIntegration, title }) => ({
+    label: title ?? name,
+    text: name,
+    isSnippet: isIntegration,
+    ...(isIntegration && { command: TRIGGER_SUGGESTION_COMMAND }),
+    kind: isIntegration ? 'Class' : 'Issue',
+    detail: isIntegration
+      ? i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.integrationDefinition', {
+          defaultMessage: `Integration`,
+        })
+      : i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.sourceDefinition', {
+          defaultMessage: `Index`,
+        }),
     sortText: 'A',
   }));
 

packages/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import type { ESQLAstItem, ESQLCommand, ESQLFunction } from '@kbn/esql-ast';
+import type { ESQLAstItem, ESQLCommand, ESQLFunction, ESQLSource } from '@kbn/esql-ast';
 import { FunctionDefinition } from '../definitions/types';
 import { getFunctionDefinition, isAssignment, isFunctionItem } from '../shared/helpers';
 
@@ -63,3 +63,11 @@ export function getQueryForFields(queryString: string, commands: ESQLCommand[])
     ? ''
     : queryString;
 }
+
+export function getSourcesFromCommands(commands: ESQLCommand[], sourceType: 'index' | 'policy') {
+  const fromCommand = commands.find(({ name }) => name === 'from');
+  const args = (fromCommand?.args ?? []) as ESQLSource[];
+  const sources = args.filter((arg) => arg.sourceType === sourceType);
+
+  return sources.length === 1 ? sources[0] : undefined;
+}

packages/kbn-esql-validation-autocomplete/src/shared/types.ts

@@ -11,7 +11,15 @@ type CallbackFn<Options = {}, Result = string> = (ctx?: Options) => Result[] | P
 
 /** @public **/
 export interface ESQLCallbacks {
-  getSources?: CallbackFn<{}, { name: string; hidden: boolean }>;
+  getSources?: CallbackFn<
+    {},
+    {
+      name: string;
+      hidden: boolean;
+      title?: string;
+      dataStreams?: Array<{ name: string; title?: string }>;
+    }
+  >;
   getFieldsFor?: CallbackFn<{ query: string }, { name: string; type: string }>;
   getPolicies?: CallbackFn<
     {},

packages/kbn-text-based-editor/src/helpers.ts

@@ -9,12 +9,27 @@
 import { useRef } from 'react';
 import useDebounce from 'react-use/lib/useDebounce';
 import { monaco } from '@kbn/monaco';
+import type { CoreStart } from '@kbn/core/public';
 import { i18n } from '@kbn/i18n';
 import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
 import type { MapCache } from 'lodash';
 
 export type MonacoMessage = monaco.editor.IMarkerData;
 
+interface IntegrationsResponse {
+  items: Array<{
+    name: string;
+    title?: string;
+    dataStreams: Array<{
+      name: string;
+      title?: string;
+    }>;
+  }>;
+}
+
+const INTEGRATIONS_API = '/api/fleet/epm/packages/installed';
+const API_VERSION = '2023-10-31';
+
 export const useDebounceWithOptions = (
   fn: Function,
   { skipFirstRender }: { skipFirstRender: boolean } = { skipFirstRender: false },
@@ -227,10 +242,38 @@ export const clearCacheWhenOld = (cache: MapCache, esqlQuery: string) => {
   }
 };
 
-export const getESQLSources = async (dataViews: DataViewsPublicPluginStart) => {
-  const [remoteIndices, localIndices] = await Promise.all([
+const getIntegrations = async (core: CoreStart) => {
+  const fleetCapabilities = core.application.capabilities.fleet;
+  if (!fleetCapabilities?.read) {
+    return [];
+  }
+  // Ideally we should use the Fleet plugin constants to fetch the integrations
+  // import { EPM_API_ROUTES, API_VERSIONS } from '@kbn/fleet-plugin/common';
+  // but it complicates things as we need to use an x-pack plugin as dependency to get 2 constants
+  // and this needs to be done in various places in the codebase which use the editor
+  // https://github.com/elastic/kibana/issues/186061
+  const response = (await core.http
+    .get(INTEGRATIONS_API, { query: undefined, version: API_VERSION })
+    .catch((error) => {
+      // eslint-disable-next-line no-console
+      console.error('Failed to fetch integrations', error);
+    })) as IntegrationsResponse;
+
+  return (
+    response?.items?.map((source) => ({
+      name: source.name,
+      hidden: false,
+      title: source.title,
+      dataStreams: source.dataStreams,
+    })) ?? []
+  );
+};
+
+export const getESQLSources = async (dataViews: DataViewsPublicPluginStart, core: CoreStart) => {
+  const [remoteIndices, localIndices, integrations] = await Promise.all([
     getRemoteIndicesList(dataViews),
     getIndicesList(dataViews),
+    getIntegrations(core),
   ]);
-  return [...localIndices, ...remoteIndices];
+  return [...localIndices, ...remoteIndices, ...integrations];
 };

packages/kbn-text-based-editor/src/text_based_languages_editor.tsx

@@ -391,7 +391,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
 
   const { cache: dataSourcesCache, memoizedSources } = useMemo(() => {
     const fn = memoize(
-      (...args: [DataViewsPublicPluginStart]) => ({
+      (...args: [DataViewsPublicPluginStart, CoreStart]) => ({
         timestamp: Date.now(),
         result: getESQLSources(...args),
       }),
@@ -405,7 +405,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
     const callbacks: ESQLCallbacks = {
       getSources: async () => {
         clearCacheWhenOld(dataSourcesCache, queryString);
-        const sources = await memoizedSources(dataViews).result;
+        const sources = await memoizedSources(dataViews, core).result;
         return sources;
       },
       getFieldsFor: async ({ query: queryToExecute }: { query?: string } | undefined = {}) => {
@@ -445,6 +445,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
     memoizedSources,
     dataSourcesCache,
     dataViews,
+    core,
     esqlFieldsCache,
     memoizedFieldsFromESQL,
     expressions,