Merge branch 'main' of github.com:elastic/kibana into refactor-common…

…-svcs

.buildkite/scripts/common/setup_job_env.sh

@@ -8,6 +8,8 @@ if [[ "$(type -t vault_get)" != "function" ]]; then
   source .buildkite/scripts/common/vault_fns.sh
 fi
 
+source .buildkite/scripts/common/util.sh
+
 # Set up general-purpose tokens and credentials
 {
   BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)"
@@ -18,9 +20,12 @@ fi
 
   KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
   KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
-  if (command -v docker && docker version) &> /dev/null; then
-    echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-  fi
+  function docker_login() {
+      if (command -v docker && docker version) &> /dev/null; then
+        echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+      fi
+  }
+  retry 5 15 docker_login
 }
 
 # Set up a custom ES Snapshot Manifest if one has been specified for this build

.buildkite/scripts/common/util.sh

@@ -178,3 +178,32 @@ print_if_dry_run() {
     echo "DRY_RUN is enabled."
   fi
 }
+
+docker_with_retry () {
+  cmd=$1
+  shift
+  args=("$@")
+  attempt=0
+  max_retries=5
+  sleep_time=15
+
+  while true
+  do
+    attempt=$((attempt+1))
+
+    if [ $attempt -gt $max_retries ]
+    then
+      echo "Docker $cmd retries exceeded, aborting."
+      exit 1
+    fi
+
+    if docker "$cmd" "${args[@]}"
+    then
+      echo "Docker $cmd successful."
+      break
+    else
+      echo "Docker $cmd unsuccessful, attempt '$attempt'... Retrying in $sleep_time"
+      sleep $sleep_time
+    fi
+  done
+}

.buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh

@@ -8,6 +8,7 @@ fi
 .buildkite/scripts/bootstrap.sh
 
 source .buildkite/scripts/steps/artifacts/env.sh
+source .buildkite/scripts/common/util.sh
 
 GIT_ABBREV_COMMIT=${BUILDKITE_COMMIT:0:12}
 KIBANA_IMAGE_TAG="sec-sol-qg-$GIT_ABBREV_COMMIT"
@@ -23,7 +24,7 @@ if docker manifest inspect $KIBANA_IMAGE &> /dev/null; then
   exit 0
 fi
 
-docker pull $KIBANA_BASE_IMAGE:latest
+docker_with_retry pull $KIBANA_BASE_IMAGE:latest
 
 echo "--- Build images"
 node scripts/build \
@@ -50,8 +51,8 @@ docker load < "target/kibana-serverless-$BASE_VERSION-docker-image-aarch64.tar.g
 docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-arm64"
 
 echo "--- Push images"
-docker image push "$KIBANA_IMAGE-arm64"
-docker image push "$KIBANA_IMAGE-amd64"
+docker_with_retry push "$KIBANA_IMAGE-arm64"
+docker_with_retry push "$KIBANA_IMAGE-amd64"
 
 echo "--- Create and push manifests"
 docker manifest create \

.buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source .buildkite/scripts/common/util.sh
+
 if [ "$KIBANA_MKI_QUALITY_GATE" == "1" ]; then
     echo "Triggered by quality gate!"
     triggered_by="Serverless Quality Gate."
@@ -17,11 +19,11 @@ else
     KBN_IMAGE=${KIBANA_LATEST}
 fi
 
-docker pull ${KBN_IMAGE}
+docker_with_retry pull ${KBN_IMAGE}
 build_date=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.build-date"')
 vcs_ref=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.vcs-ref"')
 vcs_url=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.vcs-url"')
-version=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.version"')   
+version=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.version"')
 
 markdown_text="""
 #### $triggered_by
@@ -32,9 +34,9 @@ markdown_text="""
 ---
 
 #### Kibana Container Metadata
-- Build Date            : $build_date 
-- Github Commit Hash    : $vcs_ref 
-- Github Repo           : $vcs_url 
+- Build Date            : $build_date
+- Github Commit Hash    : $vcs_ref
+- Github Repo           : $vcs_url
 - Version               : $version
 """
-echo "${markdown_text//[*\\_]/\\&}" | buildkite-agent annotate --style "info"
+echo "${markdown_text//[*\\_]/\\&}" | buildkite-agent annotate --style "info"

.buildkite/scripts/steps/artifacts/cloud.sh

@@ -28,7 +28,7 @@ docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE"
 if  docker manifest inspect $KIBANA_TEST_IMAGE &> /dev/null; then
   echo "Cloud image already exists, skipping docker push"
 else
-  docker image push "$KIBANA_TEST_IMAGE"
+  docker_with_retry push "$KIBANA_TEST_IMAGE"
 fi
 
 echo "--- Create deployment"

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -6,6 +6,8 @@ set -euo pipefail
 
 source .buildkite/scripts/steps/artifacts/env.sh
 
+source .buildkite/scripts/common/util.sh
+
 GIT_ABBREV_COMMIT=${BUILDKITE_COMMIT:0:12}
 if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
   KIBANA_IMAGE_TAG="git-$GIT_ABBREV_COMMIT"
@@ -50,8 +52,8 @@ if [[ "$SKIP_BUILD" == "false" ]]; then
   docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-arm64"
 
   echo "--- Push images"
-  docker image push "$KIBANA_IMAGE-arm64"
-  docker image push "$KIBANA_IMAGE-amd64"
+  docker_with_retry push "$KIBANA_IMAGE-arm64"
+  docker_with_retry push "$KIBANA_IMAGE-amd64"
 
   echo "--- Create and push manifests"
   docker manifest create \

.buildkite/scripts/steps/artifacts/publish.sh

@@ -54,7 +54,7 @@ chmod -R a+r target/*
 chmod -R a+w target
 
 echo "--- Pull latest Release Manager CLI"
-docker pull docker.elastic.co/infra/release-manager:latest
+docker_with_retry pull docker.elastic.co/infra/release-manager:latest
 
 echo "--- Publish artifacts"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] || [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then

.buildkite/scripts/steps/demo_env/es_and_init.sh

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 source "$(dirname "${0}")/config.sh"
+source "$(dirname "${0}")/../../common/util.sh"
 
 "$(dirname "${0}")/auth.sh"
 
@@ -16,7 +17,7 @@ DOCKER_EXPORT_URL=$(curl https://storage.googleapis.com/kibana-ci-es-snapshots-d
 curl "$DOCKER_EXPORT_URL" > target/elasticsearch-docker.tar.gz
 docker load < target/elasticsearch-docker.tar.gz
 docker tag "docker.elastic.co/elasticsearch/elasticsearch:$DEPLOYMENT_VERSION-SNAPSHOT" "$ES_IMAGE"
-docker push "$ES_IMAGE"
+docker_with_retry push "$ES_IMAGE"
 
 echo '--- Prepare yaml'
 

.buildkite/scripts/steps/demo_env/kibana.sh

@@ -4,6 +4,8 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
+source .buildkite/scripts/common/util.sh
+
 source "$(dirname "${0}")/config.sh"
 
 export KIBANA_IMAGE="gcr.io/elastic-kibana-184716/demo/kibana:$DEPLOYMENT_NAME-$(git rev-parse HEAD)"
@@ -15,7 +17,7 @@ echo '--- Build Docker image with example plugins'
 cd target/example_plugins
 BUILT_IMAGE="docker.elastic.co/kibana/kibana:$DEPLOYMENT_VERSION-SNAPSHOT"
 docker build --build-arg BASE_IMAGE="$BUILT_IMAGE" -t "$KIBANA_IMAGE" -f "$KIBANA_DIR/.buildkite/scripts/steps/demo_env/Dockerfile" .
-docker push "$KIBANA_IMAGE"
+docker_with_retry push "$KIBANA_IMAGE"
 cd -
 
 "$(dirname "${0}")/auth.sh"

.buildkite/scripts/steps/es_serverless/annotate_runtime_parameters.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+source .buildkite/scripts/common/util.sh
+
 KIBANA_GITHUB_URL="https://github.com/elastic/kibana"
 ES_SERVERLESS_GITHUB_URL="https://github.com/elastic/elasticsearch-serverless"
 
@@ -15,7 +17,7 @@ fi
 
 # Pull the target image
 if [[ $ES_SERVERLESS_IMAGE != *":git-"* ]]; then
-  docker pull "$ES_SERVERLESS_IMAGE"
+  docker_with_retry pull "$ES_SERVERLESS_IMAGE"
   ES_SERVERLESS_VERSION=$(docker inspect --format='{{json .Config.Labels}}' "$ES_SERVERLESS_IMAGE" | jq -r '.["org.opencontainers.image.revision"]' | cut -c1-12)
 
   IMAGE_WITHOUT_TAG=$(echo "$ES_SERVERLESS_IMAGE" | cut -d: -f1)

.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh

@@ -31,18 +31,18 @@ ARM_64_DIGEST=$(jq -r '.manifests[] | select(.platform.architecture == "arm64")
 AMD_64_DIGEST=$(jq -r '.manifests[] | select(.platform.architecture == "amd64") | .digest' manifests.json)
 
 echo docker pull --platform linux/arm64 "$SOURCE_IMAGE@$ARM_64_DIGEST"
-docker pull --platform linux/arm64 "$SOURCE_IMAGE@$ARM_64_DIGEST"
+docker_with_retry pull --platform linux/arm64 "$SOURCE_IMAGE@$ARM_64_DIGEST"
 echo linux/arm64 image pulled, with digest: $ARM_64_DIGEST
 
 echo docker pull --platform linux/amd64 "$SOURCE_IMAGE@$AMD_64_DIGEST"
-docker pull --platform linux/amd64 "$SOURCE_IMAGE@$AMD_64_DIGEST"
+docker_with_retry pull --platform linux/amd64 "$SOURCE_IMAGE@$AMD_64_DIGEST"
 echo linux/amd64 image pulled, with digest: $AMD_64_DIGEST
 
 docker tag "$SOURCE_IMAGE@$ARM_64_DIGEST" "$TARGET_IMAGE-arm64"
 docker tag "$SOURCE_IMAGE@$AMD_64_DIGEST" "$TARGET_IMAGE-amd64"
 
-docker push "$TARGET_IMAGE-arm64"
-docker push "$TARGET_IMAGE-amd64"
+docker_with_retry push "$TARGET_IMAGE-arm64"
+docker_with_retry push "$TARGET_IMAGE-amd64"
 
 docker manifest rm "$TARGET_IMAGE" || echo "Nothing to delete"
 

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -93,7 +93,7 @@ set +e
   echo $ES_CLOUD_ID $ES_CLOUD_VERSION $KIBANA_ES_CLOUD_VERSION $KIBANA_ES_CLOUD_IMAGE
   docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
 
-  docker image push "$KIBANA_ES_CLOUD_IMAGE"
+  docker_with_retry push "$KIBANA_ES_CLOUD_IMAGE"
 
   export ELASTICSEARCH_CLOUD_IMAGE="$KIBANA_ES_CLOUD_IMAGE"
   export ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM="$(docker images "$KIBANA_ES_CLOUD_IMAGE" --format "{{.Digest}}")"

packages/kbn-unified-data-table/src/components/data_table.scss

@@ -63,6 +63,9 @@
     border-left: 0;
     border-right: 0;
   }
+  .euiDataGridRowCell.euiDataGridRowCell--controlColumn[data-gridcell-column-id='additionalRowControl_menuControl'] .euiDataGridRowCell__content {
+    padding-bottom: 0;
+  }
 
   .euiDataGridHeaderCell.euiDataGridHeaderCell--controlColumn[data-gridcell-column-id='select'] {
     padding-left: $euiSizeXS;

src/plugins/discover/public/application/context/context_app.test.tsx

@@ -24,11 +24,13 @@ import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
 import { dataPluginMock } from '@kbn/data-plugin/public/mocks';
 import type { HistoryLocationState } from '../../build_services';
 import { createSearchSessionMock } from '../../__mocks__/search_session';
+import { createDiscoverServicesMock } from '../../__mocks__/services';
 
 const mockFilterManager = createFilterManagerMock();
 const mockNavigationPlugin = {
   ui: { TopNavMenu: mockTopNavMenu, AggregateQueryTopNavMenu: mockTopNavMenu },
 };
+const discoverServices = createDiscoverServicesMock();
 
 describe('ContextApp test', () => {
   const { history } = createSearchSessionMock();
@@ -53,6 +55,7 @@ describe('ContextApp test', () => {
     toastNotifications: { addDanger: () => {} },
     navigation: mockNavigationPlugin,
     core: {
+      ...discoverServices.core,
       executionContext: {
         set: jest.fn(),
       },
@@ -75,6 +78,7 @@ describe('ContextApp test', () => {
     },
     contextLocator: { getRedirectUrl: jest.fn(() => '') },
     singleDocLocator: { getRedirectUrl: jest.fn(() => '') },
+    profilesManager: discoverServices.profilesManager,
   } as unknown as DiscoverServices;
 
   const defaultProps = {

src/plugins/discover/public/application/context/context_app_content.tsx

@@ -40,6 +40,7 @@ import { DocTableContext } from '../../components/doc_table/doc_table_context';
 import { useDiscoverServices } from '../../hooks/use_discover_services';
 import { DiscoverGridFlyout } from '../../components/discover_grid_flyout';
 import { onResizeGridColumn } from '../../utils/on_resize_grid_column';
+import { useProfileAccessor } from '../../context_awareness';
 
 export interface ContextAppContentProps {
   columns: string[];
@@ -159,6 +160,12 @@ export function ContextAppContent({
     [grid, setAppState]
   );
 
+  const getCellRenderersAccessor = useProfileAccessor('getCellRenderers');
+  const cellRenderers = useMemo(() => {
+    const getCellRenderers = getCellRenderersAccessor(() => ({}));
+    return getCellRenderers();
+  }, [getCellRenderersAccessor]);
+
   return (
     <Fragment>
       <WrapperWithPadding>
@@ -222,6 +229,7 @@ export function ContextAppContent({
               configHeaderRowHeight={3}
               settings={grid}
               onResize={onResize}
+              externalCustomRenderers={cellRenderers}
             />
           </CellActionsProvider>
         </div>

src/plugins/discover/public/application/context/services/anchor.ts

@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-import { lastValueFrom } from 'rxjs';
+import { firstValueFrom, lastValueFrom } from 'rxjs';
 import { i18n } from '@kbn/i18n';
 import { ISearchSource, EsQuerySortValue } from '@kbn/data-plugin/public';
 import type { DataView } from '@kbn/data-views-plugin/public';
@@ -14,6 +14,7 @@ import { buildDataTableRecord } from '@kbn/discover-utils';
 import type { DataTableRecord, EsHitRecord } from '@kbn/discover-utils/types';
 import type { SearchResponseWarning } from '@kbn/search-response-warnings';
 import type { DiscoverServices } from '../../../build_services';
+import { createDataViewDataSource } from '../../../../common/data_sources';
 
 export async function fetchAnchor(
   anchorId: string,
@@ -26,6 +27,16 @@ export async function fetchAnchor(
   anchorRow: DataTableRecord;
   interceptedWarnings: SearchResponseWarning[];
 }> {
+  const { core, profilesManager } = services;
+
+  const solutionNavId = await firstValueFrom(core.chrome.getActiveSolutionNavId$());
+  await profilesManager.resolveRootProfile({ solutionNavId });
+  await profilesManager.resolveDataSourceProfile({
+    dataSource: dataView?.id ? createDataViewDataSource({ dataViewId: dataView.id }) : undefined,
+    dataView,
+    query: { query: '', language: 'kuery' },
+  });
+
   updateSearchSource(searchSource, anchorId, sort, useNewFieldsApi, dataView);
 
   const adapter = new RequestAdapter();
@@ -55,7 +66,9 @@ export async function fetchAnchor(
   });
 
   return {
-    anchorRow: buildDataTableRecord(doc, dataView, true),
+    anchorRow: profilesManager.resolveDocumentProfile({
+      record: buildDataTableRecord(doc, dataView, true),
+    }),
     interceptedWarnings,
   };
 }

src/plugins/discover/public/application/context/utils/fetch_hits_in_interval.ts

@@ -46,6 +46,7 @@ export async function fetchHitsInInterval(
   rows: DataTableRecord[];
   interceptedWarnings: SearchResponseWarning[];
 }> {
+  const { profilesManager } = services;
   const range: RangeQuery = {
     format: 'strict_date_optional_time',
   };
@@ -96,7 +97,11 @@ export async function fetchHitsInInterval(
 
   const { rawResponse } = await lastValueFrom(fetch$);
   const dataView = searchSource.getField('index');
-  const rows = rawResponse.hits?.hits.map((hit) => buildDataTableRecord(hit, dataView!));
+  const rows = rawResponse.hits?.hits.map((hit) =>
+    profilesManager.resolveDocumentProfile({
+      record: buildDataTableRecord(hit, dataView!),
+    })
+  );
   const interceptedWarnings: SearchResponseWarning[] = [];
   services.data.search.showWarnings(adapter, (warning) => {
     interceptedWarnings.push(warning);

src/plugins/discover/public/application/doc/components/doc.test.tsx

@@ -19,7 +19,9 @@ import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
 import { setUnifiedDocViewerServices } from '@kbn/unified-doc-viewer-plugin/public/plugin';
 import { mockUnifiedDocViewerServices } from '@kbn/unified-doc-viewer-plugin/public/__mocks__';
 import type { UnifiedDocViewerServices } from '@kbn/unified-doc-viewer-plugin/public/types';
+import { createDiscoverServicesMock } from '../../../__mocks__/services';
 
+const discoverServices = createDiscoverServicesMock();
 const mockSearchApi = jest.fn();
 
 beforeEach(() => {
@@ -68,6 +70,8 @@ async function mountDoc(update = false) {
     },
     locator: { getUrl: jest.fn(() => Promise.resolve('mock-url')) },
     chrome: { setBreadcrumbs: jest.fn() },
+    profilesManager: discoverServices.profilesManager,
+    core: discoverServices.core,
   };
   setUnifiedDocViewerServices({
     ...mockUnifiedDocViewerServices,