[Discover] Add support for unmapped fields using the fields API

src/plugins/discover/public/application/angular/context_app.js

@@ -50,14 +50,15 @@ getAngularModule().directive('contextApp', function ContextApp() {
 
 function ContextAppController($scope, Private) {
   const { filterManager, indexPatterns, uiSettings, navigation } = getServices();
+  const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE);
   const queryParameterActions = getQueryParameterActions(filterManager, indexPatterns);
   const queryActions = Private(QueryActionsProvider);
-  const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE);
   this.state = createInitialState(
     parseInt(uiSettings.get(CONTEXT_STEP_SETTING), 10),
     getFirstSortableField(this.indexPattern, uiSettings.get(CONTEXT_TIE_BREAKER_FIELDS_SETTING)),
     useNewFieldsApi
   );
+  this.state.useNewFieldsApi = useNewFieldsApi;
   this.topNavMenu = navigation.ui.TopNavMenu;
 
   this.actions = _.mapValues(

src/plugins/discover/public/application/angular/discover.js

@@ -748,6 +748,21 @@ function discoverController($route, $scope, Promise) {
     history.push('/');
   };
 
+  const showUnmappedFieldsDefaultValue = $scope.useNewFieldsApi && !!$scope.opts.savedSearch.pre712;
+  let showUnmappedFields = showUnmappedFieldsDefaultValue;
+
+  const onChangeUnmappedFields = (value) => {
+    showUnmappedFields = value;
+    $scope.unmappedFieldsConfig.showUnmappedFields = value;
+    $scope.fetch();
+  };
+
+  $scope.unmappedFieldsConfig = {
+    showUnmappedFieldsDefaultValue,
+    showUnmappedFields,
+    onChangeUnmappedFields,
+  };
+
   $scope.updateDataSource = () => {
     const { indexPattern, searchSource, useNewFieldsApi } = $scope;
     const { columns, sort } = $scope.state;
@@ -757,6 +772,7 @@ function discoverController($route, $scope, Promise) {
       sort,
       columns,
       useNewFieldsApi,
+      showUnmappedFields,
     });
     return Promise.resolve();
   };

src/plugins/discover/public/application/angular/discover_legacy.html

@@ -30,6 +30,7 @@
     update-query="handleRefresh"
     update-saved-query-id="updateSavedQueryId"
     use-new-fields-api="useNewFieldsApi"
+    unmapped-fields-config="unmappedFieldsConfig"
   >
   </discover>
 </discover-app>

src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx

@@ -5,7 +5,6 @@
  * compliance with, at your election, the Elastic License or the Server Side
  * Public License, v 1.
  */
-
 import { i18n } from '@kbn/i18n';
 import { IndexPattern } from '../../../../../kibana_services';
 

src/plugins/discover/public/application/components/create_discover_directive.ts

@@ -40,5 +40,6 @@ export function createDiscoverDirective(reactDirective: any) {
     ['topNavMenu', { watchDepth: 'reference' }],
     ['updateQuery', { watchDepth: 'reference' }],
     ['updateSavedQueryId', { watchDepth: 'reference' }],
+    ['unmappedFieldsConfig', { watchDepth: 'value' }],
   ]);
 }

src/plugins/discover/public/application/components/discover.tsx

@@ -80,6 +80,7 @@ export function Discover({
   topNavMenu,
   updateQuery,
   updateSavedQueryId,
+  unmappedFieldsConfig,
 }: DiscoverProps) {
   const scrollableDesktop = useRef<HTMLDivElement>(null);
   const collapseIcon = useRef<HTMLButtonElement>(null);
@@ -139,6 +140,7 @@ export function Discover({
                 setIndexPattern={setIndexPattern}
                 isClosed={isSidebarClosed}
                 trackUiMetric={trackUiMetric}
+                unmappedFieldsConfig={unmappedFieldsConfig}
                 useNewFieldsApi={useNewFieldsApi}
               />
             </EuiFlexItem>

src/plugins/discover/public/application/components/sidebar/discover_field_search.test.tsx

@@ -136,4 +136,22 @@ describe('DiscoverFieldSearch', () => {
     popover = component.find(EuiPopover);
     expect(popover.prop('isOpen')).toBe(false);
   });
+
+  test('unmapped fields', () => {
+    const onChangeUnmappedFields = jest.fn();
+    const componentProps = {
+      ...defaultProps,
+      showUnmappedFields: true,
+      useNewFieldsApi: false,
+      onChangeUnmappedFields,
+    };
+    const component = mountComponent(componentProps);
+    const btn = findTestSubject(component, 'toggleFieldFilterButton');
+    btn.simulate('click');
+    const unmappedFieldsSwitch = findTestSubject(component, 'unmappedFieldsSwitch');
+    act(() => {
+      unmappedFieldsSwitch.simulate('click');
+    });
+    expect(onChangeUnmappedFields).toHaveBeenCalledWith(false);
+  });
 });

src/plugins/discover/public/application/components/sidebar/discover_field_search.tsx

@@ -27,6 +27,8 @@ import {
   EuiOutsideClickDetector,
   EuiFilterButton,
   EuiSpacer,
+  EuiIcon,
+  EuiToolTip,
 } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
 
@@ -35,6 +37,7 @@ export interface State {
   aggregatable: string;
   type: string;
   missing: boolean;
+  unmappedFields: boolean;
   [index: string]: string | boolean;
 }
 
@@ -53,13 +56,36 @@ export interface Props {
    * types for the type filter
    */
   types: string[];
+
+  /**
+   * use new fields api
+   */
+  useNewFieldsApi?: boolean;
+
+  /**
+   * callback funtion to change the value of unmapped fields switch
+   * @param value new value to set
+   */
+  onChangeUnmappedFields?: (value: boolean) => void;
+
+  /**
+   * should unmapped fields switch be rendered
+   */
+  showUnmappedFields?: boolean;
 }
 
 /**
  * Component is Discover's side bar to  search of available fields
  * Additionally there's a button displayed that allows the user to show/hide more filter fields
  */
-export function DiscoverFieldSearch({ onChange, value, types }: Props) {
+export function DiscoverFieldSearch({
+  onChange,
+  value,
+  types,
+  useNewFieldsApi,
+  showUnmappedFields,
+  onChangeUnmappedFields,
+}: Props) {
   const searchPlaceholder = i18n.translate('discover.fieldChooser.searchPlaceHolder', {
     defaultMessage: 'Search field names',
   });
@@ -85,6 +111,7 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) {
     aggregatable: 'any',
     type: 'any',
     missing: true,
+    unmappedFields: !!showUnmappedFields,
   });
 
   if (typeof value !== 'string') {
@@ -154,6 +181,14 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) {
     handleValueChange('missing', missingValue);
   };
 
+  const handleUnmappedFieldsChange = (e: EuiSwitchEvent) => {
+    const unmappedFieldsValue = e.target.checked;
+    handleValueChange('unmappedFields', unmappedFieldsValue);
+    if (onChangeUnmappedFields) {
+      onChangeUnmappedFields(unmappedFieldsValue);
+    }
+  };
+
   const buttonContent = (
     <EuiFilterButton
       aria-label={filterBtnAriaLabel}
@@ -226,6 +261,51 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) {
     );
   };
 
+  const footer = () => {
+    if (!showUnmappedFields && useNewFieldsApi) {
+      return null;
+    }
+    return (
+      <EuiPopoverFooter>
+        {showUnmappedFields ? (
+          <EuiFlexGroup>
+            <EuiFlexItem component="span">
+              <EuiSwitch
+                label={i18n.translate('discover.fieldChooser.filter.showUnmappedFields', {
+                  defaultMessage: 'Show unmapped fields',
+                })}
+                checked={values.unmappedFields}
+                onChange={handleUnmappedFieldsChange}
+                data-test-subj="unmappedFieldsSwitch"
+              />
+            </EuiFlexItem>
+            <EuiFlexItem component="span" grow={false}>
+              <EuiToolTip
+                position="right"
+                content={i18n.translate('discover.fieldChooser.filter.unmappedFieldsWarning', {
+                  defaultMessage:
+                    'Unmapped fields will soon be deprecated. We suggest keeping this switch off.',
+                })}
+              >
+                <EuiIcon type="alert" />
+              </EuiToolTip>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        ) : null}
+        {useNewFieldsApi ? null : (
+          <EuiSwitch
+            label={i18n.translate('discover.fieldChooser.filter.hideMissingFieldsLabel', {
+              defaultMessage: 'Hide missing fields',
+            })}
+            checked={values.missing}
+            onChange={handleMissingChange}
+            data-test-subj="missingSwitch"
+          />
+        )}
+      </EuiPopoverFooter>
+    );
+  };
+
   const selectionPanel = (
     <div className="dscFieldSearch__formWrapper">
       <EuiForm data-test-subj="filterSelectionPanel">
@@ -277,16 +357,7 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) {
               })}
             </EuiPopoverTitle>
             {selectionPanel}
-            <EuiPopoverFooter>
-              <EuiSwitch
-                label={i18n.translate('discover.fieldChooser.filter.hideMissingFieldsLabel', {
-                  defaultMessage: 'Hide missing fields',
-                })}
-                checked={values.missing}
-                onChange={handleMissingChange}
-                data-test-subj="missingSwitch"
-              />
-            </EuiPopoverFooter>
+            {footer()}
           </EuiPopover>
         </EuiFilterGroup>
       </EuiOutsideClickDetector>

src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx

@@ -104,6 +104,27 @@ export interface DiscoverSidebarProps {
    * Shows index pattern and a button that displays the sidebar in a flyout
    */
   useFlyout?: boolean;
+
+  /**
+   * an object containing properties for proper handling of unmapped fields in the UI
+   */
+  unmappedFieldsConfig?: {
+    /**
+     * callback funtction to change the value of `showUnmappedFields` flag
+     * @param value new value to set
+     */
+    onChangeUnmappedFields: (value: boolean) => void;
+    /**
+     * determines whether to display unmapped fields
+     * configurable through the switch in the UI
+     */
+    showUnmappedFields: boolean;
+    /**
+     * determines if we should display an option to toggle showUnmappedFields value in the first place
+     * this value is not configurable through the UI
+     */
+    showUnmappedFieldsDefaultValue: boolean;
+  };
 }
 
 export function DiscoverSidebar({
@@ -123,6 +144,7 @@ export function DiscoverSidebar({
   trackUiMetric,
   useNewFieldsApi = false,
   useFlyout = false,
+  unmappedFieldsConfig,
 }: DiscoverSidebarProps) {
   const [fields, setFields] = useState<IndexPatternField[] | null>(null);
 
@@ -145,14 +167,30 @@ export function DiscoverSidebar({
   );
 
   const popularLimit = services.uiSettings.get(FIELDS_LIMIT_SETTING);
-
   const {
     selected: selectedFields,
     popular: popularFields,
     unpopular: unpopularFields,
   } = useMemo(
-    () => groupFields(fields, columns, popularLimit, fieldCounts, fieldFilter, useNewFieldsApi),
-    [fields, columns, popularLimit, fieldCounts, fieldFilter, useNewFieldsApi]
+    () =>
+      groupFields(
+        fields,
+        columns,
+        popularLimit,
+        fieldCounts,
+        fieldFilter,
+        useNewFieldsApi,
+        !!unmappedFieldsConfig?.showUnmappedFields
+      ),
+    [
+      fields,
+      columns,
+      popularLimit,
+      fieldCounts,
+      fieldFilter,
+      useNewFieldsApi,
+      unmappedFieldsConfig?.showUnmappedFields,
+    ]
   );
 
   const fieldTypes = useMemo(() => {
@@ -239,6 +277,9 @@ export function DiscoverSidebar({
               onChange={onChangeFieldSearch}
               value={fieldFilter.name}
               types={fieldTypes}
+              useNewFieldsApi={useNewFieldsApi}
+              onChangeUnmappedFields={unmappedFieldsConfig?.onChangeUnmappedFields}
+              showUnmappedFields={unmappedFieldsConfig?.showUnmappedFieldsDefaultValue}
             />
           </form>
         </EuiFlexItem>

src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.test.tsx

@@ -15,7 +15,7 @@ import realHits from 'fixtures/real_hits.js';
 import stubbedLogstashFields from 'fixtures/logstash_fields';
 import { mountWithIntl } from '@kbn/test/jest';
 import React from 'react';
-import { DiscoverSidebarProps } from './discover_sidebar';
+import { DiscoverSidebar, DiscoverSidebarProps } from './discover_sidebar';
 import { coreMock } from '../../../../../../core/public/mocks';
 import { IndexPatternAttributes } from '../../../../../data/common';
 import { getStubIndexPattern } from '../../../../../data/public/test_utils';
@@ -131,4 +131,16 @@ describe('discover responsive sidebar', function () {
     findTestSubject(comp, 'plus-extension-gif').simulate('click');
     expect(props.onAddFilter).toHaveBeenCalled();
   });
+  it('renders sidebar with unmapped fields config', function () {
+    const unmappedFieldsConfig = {
+      onChangeUnmappedFields: jest.fn(),
+      showUnmappedFields: false,
+      showUnmappedFieldsDefaultValue: false,
+    };
+    const componentProps = { ...props, unmappedFieldsConfig };
+    const component = mountWithIntl(<DiscoverSidebarResponsive {...componentProps} />);
+    const discoverSidebar = component.find(DiscoverSidebar);
+    expect(discoverSidebar).toHaveLength(1);
+    expect(discoverSidebar.props().unmappedFieldsConfig).toEqual(unmappedFieldsConfig);
+  });
 });

src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx

@@ -97,6 +97,27 @@ export interface DiscoverSidebarResponsiveProps {
    * Read from the Fields API
    */
   useNewFieldsApi?: boolean;
+
+  /**
+   * an object containing properties for proper handling of unmapped fields in the UI
+   */
+  unmappedFieldsConfig?: {
+    /**
+     * callback function to change the value of `showUnmappedFields` flag
+     * @param value new value to set
+     */
+    onChangeUnmappedFields: (value: boolean) => void;
+    /**
+     * determines whether to display unmapped fields
+     * configurable through the switch in the UI
+     */
+    showUnmappedFields: boolean;
+    /**
+     * determines if we should display an option to toggle showUnmappedFields value in the first place
+     * this value is not configurable through the UI
+     */
+    showUnmappedFieldsDefaultValue: boolean;
+  };
 }
 
 /**