[data views] cache field caps requests

docs/concepts/data-views.asciidoc

@@ -172,3 +172,14 @@ WARNING: Deleting a {data-source} breaks all visualizations, saved searches, and
 
 . Find the {data-source} that you want to delete, and then
 click image:management/index-patterns/images/delete.png[Delete icon] in the *Actions* column.
+
+[float]
+[[data-view-field-cache]]
+=== {data-source} field cache
+
+The browser caches {data-source} field lists for increased performance. This is particularly impactful 
+for {data-sources} with a high field count that span a large number of indices and clusters. The field 
+list is updated every couple of minutes in typical {kib} usage. Alternatively, use the refresh button on the {data-source} 
+management detail page to get an updated field list. A force reload of {kib} has the same effect. 
+
+The field list may be impacted by changes in indices and user permissions.

packages/core/http/core-http-router-server-internal/src/response.ts

@@ -17,6 +17,7 @@ import type {
   ErrorHttpResponseOptions,
   KibanaErrorResponseFactory,
   KibanaRedirectionResponseFactory,
+  KibanaNotModifiedResponseFactory,
   KibanaSuccessResponseFactory,
   KibanaResponseFactory,
   LifecycleResponseFactory,
@@ -51,6 +52,10 @@ const redirectionResponseFactory: KibanaRedirectionResponseFactory = {
   redirected: (options: RedirectResponseOptions) => new KibanaResponse(302, options.body, options),
 };
 
+const notModifiedResponseFactory: KibanaNotModifiedResponseFactory = {
+  notModified: (options: HttpResponseOptions = {}) => new KibanaResponse(304, undefined, options),
+};
+
 const errorResponseFactory: KibanaErrorResponseFactory = {
   badRequest: (options: ErrorHttpResponseOptions = {}) =>
     new KibanaResponse(400, options.body || 'Bad Request', options),
@@ -120,6 +125,7 @@ export const fileResponseFactory = {
 export const kibanaResponseFactory: KibanaResponseFactory = {
   ...successResponseFactory,
   ...redirectionResponseFactory,
+  ...notModifiedResponseFactory,
   ...errorResponseFactory,
   ...fileResponseFactory,
   custom: <T extends HttpResponsePayload | ResponseError>(

packages/core/http/core-http-router-server-internal/src/response_adapter.ts

@@ -33,7 +33,8 @@ function setHeaders(response: HapiResponseObject, headers: Record<string, string
 
 const statusHelpers = {
   isSuccess: (code: number) => code >= 100 && code < 300,
-  isRedirect: (code: number) => code >= 300 && code < 400,
+  isNotModified: (code: number) => code === 304,
+  isRedirect: (code: number) => code >= 300 && code < 400 && code !== 304,
   isError: (code: number) => code >= 400 && code < 600,
 };
 
@@ -76,7 +77,10 @@ export class HapiResponseAdapter {
     if (statusHelpers.isError(kibanaResponse.status)) {
       return this.toError(kibanaResponse);
     }
-    if (statusHelpers.isSuccess(kibanaResponse.status)) {
+    if (
+      statusHelpers.isSuccess(kibanaResponse.status) ||
+      statusHelpers.isNotModified(kibanaResponse.status)
+    ) {
       return this.toSuccess(kibanaResponse);
     }
     if (statusHelpers.isRedirect(kibanaResponse.status)) {

packages/core/http/core-http-router-server-mocks/src/router.mock.ts

@@ -121,6 +121,7 @@ const createResponseFactoryMock = (): jest.Mocked<KibanaResponseFactory> => ({
   ok: jest.fn(),
   accepted: jest.fn(),
   noContent: jest.fn(),
+  notModified: jest.fn(),
   custom: jest.fn(),
   redirected: jest.fn(),
   badRequest: jest.fn(),

packages/core/http/core-http-server/index.ts

@@ -95,6 +95,7 @@ export type {
   IKibanaSocket,
   KibanaErrorResponseFactory,
   KibanaRedirectionResponseFactory,
+  KibanaNotModifiedResponseFactory,
   KibanaSuccessResponseFactory,
   KibanaResponseFactory,
   LifecycleResponseFactory,

packages/core/http/core-http-server/src/router/index.ts

@@ -66,6 +66,7 @@ export type { IKibanaSocket } from './socket';
 export type {
   KibanaErrorResponseFactory,
   KibanaRedirectionResponseFactory,
+  KibanaNotModifiedResponseFactory,
   KibanaSuccessResponseFactory,
   KibanaResponseFactory,
   LifecycleResponseFactory,

packages/core/http/core-http-server/src/router/response.ts

@@ -95,9 +95,7 @@ export interface FileHttpResponseOptions<T extends HttpResponsePayload | Respons
  * @public
  */
 export type RedirectResponseOptions = HttpResponseOptions & {
-  headers: {
-    location: string;
-  };
+  headers?: ResponseHeaders;
 };
 
 /**

packages/core/http/core-http-server/src/router/response_factory.ts

@@ -61,6 +61,18 @@ export interface KibanaRedirectionResponseFactory {
   redirected(options: RedirectResponseOptions): IKibanaResponse;
 }
 
+/**
+ * @public
+ */
+export interface KibanaNotModifiedResponseFactory {
+  /**
+   * Content not modified.
+   * Status code: `304`.
+   * @param options - {@link HttpResponseOptions} configures HTTP response body & headers.
+   */
+  notModified(options: HttpResponseOptions): IKibanaResponse;
+}
+
 /**
  * @public
  */
@@ -200,6 +212,7 @@ export interface KibanaErrorResponseFactory {
  */
 export type KibanaResponseFactory = KibanaSuccessResponseFactory &
   KibanaRedirectionResponseFactory &
+  KibanaNotModifiedResponseFactory &
   KibanaErrorResponseFactory & {
     /**
      * Creates a response with defined status code and payload.

packages/kbn-optimizer/limits.yml

@@ -33,7 +33,7 @@ pageLoadAssetSize:
   dataViewEditor: 28082
   dataViewFieldEditor: 27000
   dataViewManagement: 5136
-  dataViews: 48300
+  dataViews: 51000
   dataVisualizer: 27530
   devTools: 38637
   discover: 99999

packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts

@@ -61,7 +61,7 @@ export async function fetchFieldExistence({
   const newFields = existingFieldList.filter((field) => !dataView.getFieldByName(field.name));
   // refresh the data view in case there are new fields
   if (newFields.length) {
-    await dataViewsService.refreshFields(dataView, false);
+    await dataViewsService.refreshFields(dataView, false, true);
   }
   const allFields = buildFieldList(dataView, metaFields);
   return {

src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx

@@ -95,6 +95,9 @@ export const EditIndexPattern = withRouter(
     const [showEditDialog, setShowEditDialog] = useState<boolean>(false);
     const [relationships, setRelationships] = useState<SavedObjectRelationWithTitle[]>([]);
     const [allowedTypes, setAllowedTypes] = useState<SavedObjectManagementTypeInfo[]>([]);
+    const [refreshCount, setRefreshCount] = useState<number>(0); // used for forcing rerender of field list
+    const [isRefreshing, setIsRefreshing] = React.useState(false);
+
     const conflictFieldsUrl = useMemo(() => {
       return setStateToKbnUrl(
         APP_STATE_STORAGE_KEY,
@@ -144,7 +147,7 @@ export const EditIndexPattern = withRouter(
       setConflictedFields(
         indexPattern.fields.getAll().filter((field) => field.type === 'conflict')
       );
-    }, [indexPattern]);
+    }, [indexPattern, refreshCount]);
 
     useEffect(() => {
       setTags(
@@ -332,6 +335,13 @@ export const EditIndexPattern = withRouter(
             setFields(indexPattern.getNonScriptedFields());
             setCompositeRuntimeFields(getCompositeRuntimeFields(indexPattern));
           }}
+          refreshIndexPatternClick={async () => {
+            setIsRefreshing(true);
+            await dataViews.refreshFields(indexPattern, false, true);
+            setRefreshCount(refreshCount + 1); // rerender field list
+            setIsRefreshing(false);
+          }}
+          isRefreshing={isRefreshing}
         />
         {displayIndexPatternEditor}
       </div>

src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx

@@ -17,7 +17,9 @@ interface IndexHeaderProps {
   setDefault?: () => void;
   editIndexPatternClick?: () => void;
   deleteIndexPatternClick?: () => void;
+  refreshIndexPatternClick?: () => void;
   canSave: boolean;
+  isRefreshing?: boolean;
 }
 
 const setDefaultAriaLabel = i18n.translate('indexPatternManagement.editDataView.setDefaultAria', {
@@ -63,6 +65,7 @@ export const IndexHeader: React.FC<IndexHeaderProps> = ({
             iconType="pencil"
             aria-label={editAriaLabel}
             data-test-subj="editIndexPatternButton"
+            color="primary"
           >
             {editTooltip}
           </EuiButton>

src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx

@@ -19,9 +19,10 @@ import {
   EuiTabbedContentTab,
   EuiSpacer,
   EuiFieldSearch,
-  EuiButton,
   EuiFilterSelectItem,
   FilterChecked,
+  EuiToolTip,
+  EuiButton,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { fieldWildcardMatcher } from '@kbn/kibana-utils-plugin/public';
@@ -60,6 +61,8 @@ interface TabsProps extends Pick<RouteComponentProps, 'history' | 'location'> {
   relationships: SavedObjectRelation[];
   allowedTypes: SavedObjectManagementTypeInfo[];
   compositeRuntimeFields: Record<string, RuntimeField>;
+  refreshIndexPatternClick: () => void;
+  isRefreshing?: boolean;
 }
 
 interface FilterItems {
@@ -139,6 +142,14 @@ const addFieldButtonLabel = i18n.translate(
   }
 );
 
+const refreshAriaLabel = i18n.translate('indexPatternManagement.editDataView.refreshAria', {
+  defaultMessage: 'Refresh',
+});
+
+const refreshTooltip = i18n.translate('indexPatternManagement.editDataView.refreshTooltip', {
+  defaultMessage: 'Refresh local copy of data view field list',
+});
+
 const SCHEMA_ITEMS: FilterItems[] = [
   {
     value: 'runtime',
@@ -159,6 +170,8 @@ export const Tabs: React.FC<TabsProps> = ({
   relationships,
   allowedTypes,
   compositeRuntimeFields,
+  refreshIndexPatternClick,
+  isRefreshing,
 }) => {
   const {
     uiSettings,
@@ -284,7 +297,9 @@ export const Tabs: React.FC<TabsProps> = ({
 
     setIndexedFieldTypes(convertToEuiFilterOptions(tempIndexedFieldTypes));
     setScriptedFieldLanguages(convertToEuiFilterOptions(tempScriptedFieldLanguages));
-  }, [indexPattern]);
+    // need to reset based on changes to fields but indexPattern is the same
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [indexPattern, fields]);
 
   const closeFieldEditor = useCallback(() => {
     if (closeEditorHandler.current) {
@@ -321,11 +336,13 @@ export const Tabs: React.FC<TabsProps> = ({
     [uiSettings]
   );
 
+  const refreshRef = useRef<HTMLButtonElement>(null);
+
   const userEditPermission = dataViews.getCanSaveSync();
   const getFilterSection = useCallback(
     (type: string) => {
       return (
-        <EuiFlexGroup>
+        <EuiFlexGroup gutterSize="m">
           <EuiFlexItem grow={true}>
             <EuiFieldSearch
               fullWidth
@@ -426,9 +443,41 @@ export const Tabs: React.FC<TabsProps> = ({
                   </EuiPopover>
                 </EuiFilterGroup>
               </EuiFlexItem>
+              <EuiFlexItem grow={false}>
+                <EuiToolTip content={<p>{refreshTooltip}</p>}>
+                  <EuiButton
+                    buttonRef={refreshRef}
+                    onClick={() => {
+                      refreshIndexPatternClick();
+                      // clear tooltip focus
+                      if (refreshRef.current) {
+                        refreshRef.current.blur();
+                      }
+                    }}
+                    iconType="refresh"
+                    aria-label={refreshAriaLabel}
+                    data-test-subj="refreshDataViewButton"
+                    isLoading={isRefreshing}
+                    isDisabled={isRefreshing}
+                    size="m"
+                    color="success"
+                    className="eui-fullWidth"
+                  >
+                    {refreshAriaLabel}
+                  </EuiButton>
+                </EuiToolTip>
+              </EuiFlexItem>
               {userEditPermission && (
                 <EuiFlexItem grow={false}>
-                  <EuiButton fill onClick={() => openFieldEditor()} data-test-subj="addField">
+                  <EuiButton
+                    size="m"
+                    onClick={() => openFieldEditor()}
+                    data-test-subj="addField"
+                    iconType="plusInCircle"
+                    aria-label={addFieldButtonLabel}
+                    color="primary"
+                    fill
+                  >
                     {addFieldButtonLabel}
                   </EuiButton>
                 </EuiFlexItem>
@@ -503,6 +552,8 @@ export const Tabs: React.FC<TabsProps> = ({
       updateFieldFilter,
       updateFieldTypeFilter,
       updateSchemaFieldTypeFilter,
+      isRefreshing,
+      refreshIndexPatternClick,
     ]
   );
 

src/plugins/data_view_management/public/components/index_pattern_table/index_pattern_table.tsx

@@ -221,7 +221,7 @@ export const IndexPatternTable = ({
         <div>
           <EuiLink
             {...reactRouterNavigate(history, `patterns/${dataView.id}`)}
-            data-test-subj={`detail-link-${dataView.name}`}
+            data-test-subj={`detail-link-${dataView.getName()}`}
           >
             {dataView.getName()}
             {dataView.name ? (

src/plugins/data_views/common/constants.ts

@@ -58,6 +58,12 @@ export const PLUGIN_NAME = 'DataViews';
  */
 export const FIELDS_FOR_WILDCARD_PATH = '/internal/data_views/_fields_for_wildcard';
 
+/**
+ * Fields path. Like fields for wildcard but GET only
+ * @public
+ */
+export const FIELDS_PATH = '/internal/data_views/fields';
+
 /**
  * Existing indices path
  * @public

src/plugins/data_views/common/data_views/data_view.ts

@@ -61,6 +61,8 @@ export class DataView extends AbstractDataView implements DataViewBase {
    */
   public flattenHit: (hit: Record<string, unknown[]>, deep?: boolean) => Record<string, unknown>;
 
+  private etag: string | undefined;
+
   /**
    * constructor
    * @param config - config data and dependencies
@@ -77,6 +79,10 @@ export class DataView extends AbstractDataView implements DataViewBase {
     this.fields.replaceAll(Object.values(spec.fields || {}));
   }
 
+  getEtag = () => this.etag;
+
+  setEtag = (etag: string | undefined) => (this.etag = etag);
+
   /**
    * Returns scripted fields
    */

src/plugins/data_views/common/data_views/data_views.test.ts

@@ -158,10 +158,23 @@ describe('IndexPatterns', () => {
 
   test('force field refresh', async () => {
     const id = '1';
+    // make sure initial load and subsequent reloads use same params
+    const args = {
+      allowHidden: undefined,
+      allowNoIndex: true,
+      indexFilter: undefined,
+      metaFields: false,
+      pattern: 'something',
+      rollupIndex: undefined,
+      type: undefined,
+    };
+
     await indexPatterns.get(id);
     expect(apiClient.getFieldsForWildcard).toBeCalledTimes(1);
+    expect(apiClient.getFieldsForWildcard).toBeCalledWith(args);
     await indexPatterns.get(id, undefined, true);
     expect(apiClient.getFieldsForWildcard).toBeCalledTimes(2);
+    expect(apiClient.getFieldsForWildcard).toBeCalledWith(args);
   });
 
   test('getFieldsForWildcard called with allowNoIndex set to true as default ', async () => {