diff --git a/fleet_packages.json b/fleet_packages.json index a6f259d19774c..7db736484db5f 100644 --- a/fleet_packages.json +++ b/fleet_packages.json @@ -24,13 +24,13 @@ [ { "name": "apm", - "version": "8.10.2-preview-1695056404", + "version": "8.10.3-preview-1695284222", "forceAlignStackVersion": true, "allowSyncToPrerelease": true }, { "name": "elastic_agent", - "version": "1.13.0" + "version": "1.13.1" }, { "name": "endpoint", diff --git a/package.json b/package.json index 4c736301b7628..7f35e37f44361 100644 --- a/package.json +++ b/package.json @@ -849,7 +849,7 @@ "del": "^6.1.0", "elastic-apm-node": "^3.49.1", "email-addresses": "^5.0.0", - "execa": "^4.0.2", + "execa": "^5.1.1", "expiry-js": "0.1.7", "extract-zip": "^2.0.1", "fast-deep-equal": "^3.1.1", diff --git a/x-pack/plugins/enterprise_search/public/applications/__mocks__/kea_logic/kibana_logic.mock.ts b/x-pack/plugins/enterprise_search/public/applications/__mocks__/kea_logic/kibana_logic.mock.ts index 2eb63fb974b44..f374fe8b60666 100644 --- a/x-pack/plugins/enterprise_search/public/applications/__mocks__/kea_logic/kibana_logic.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/__mocks__/kea_logic/kibana_logic.mock.ts @@ -61,7 +61,6 @@ export const mockKibanaValues = { setDocTitle: jest.fn(), share: sharePluginMock.createStartContract(), uiSettings: uiSettingsServiceMock.createStartContract(), - userProfile: {}, }; jest.mock('../../shared/kibana', () => ({ diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx index b82305b4c4d04..e197d0551253e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx @@ -10,6 +10,7 @@ import React, { useEffect, useState, ChangeEvent } from 'react'; import { useActions, useValues } from 'kea'; import { + EuiCallOut, EuiFieldSearch, EuiFlexGroup, EuiFlexItem, @@ -73,7 +74,7 @@ export const SearchIndexDocuments: React.FC = () => { const { makeRequest: getDocuments } = useActions(documentLogic); const { makeRequest: getMappings } = useActions(mappingLogic); - const { data, status } = useValues(documentLogic); + const { data, status, error } = useValues(documentLogic); const { data: mappingData, status: mappingStatus } = useValues(mappingLogic); const docs = data?.results?.hits.hits ?? []; @@ -83,6 +84,8 @@ export const SearchIndexDocuments: React.FC = () => { const shouldShowAccessControlSwitcher = hasDocumentLevelSecurityFeature && productFeatures.hasDocumentLevelSecurityEnabled; + const isAccessControlIndexNotFound = + shouldShowAccessControlSwitcher && error?.body?.statusCode === 404; useEffect(() => { getDocuments({ @@ -140,11 +143,29 @@ export const SearchIndexDocuments: React.FC = () => { - {docs.length === 0 && + {isAccessControlIndexNotFound && ( + +

+ {i18n.translate('xpack.enterpriseSearch.content.searchIndex.documents.noIndex', { + defaultMessage: + "An Access Control Index won't be created until you enable document-level security and run your first access control sync.", + })} +

+ + )} + {!isAccessControlIndexNotFound && + docs.length === 0 && i18n.translate('xpack.enterpriseSearch.content.searchIndex.documents.noMappings', { defaultMessage: 'No documents found for index', })} - {docs.length > 0 && ( + {!isAccessControlIndexNotFound && docs.length > 0 && ( { ? indexName : indexName.replace('search-', CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX); const { makeRequest: makeMappingRequest } = useActions(mappingsWithPropsApiLogic(indexToShow)); - const { data: mappingData } = useValues(mappingsWithPropsApiLogic(indexToShow)); + const { data: mappingData, error } = useValues(mappingsWithPropsApiLogic(indexToShow)); const shouldShowAccessControlSwitch = hasDocumentLevelSecurityFeature && productFeatures.hasDocumentLevelSecurityEnabled; + const isAccessControlIndexNotFound = + shouldShowAccessControlSwitch && error?.body?.statusCode === 404; + useEffect(() => { makeMappingRequest({ indexName: indexToShow }); }, [indexToShow, indexName]); @@ -76,9 +80,27 @@ export const SearchIndexIndexMappings: React.FC = () => { )} - - {JSON.stringify(mappingData, null, 2)} - + {isAccessControlIndexNotFound ? ( + +

+ {i18n.translate('xpack.enterpriseSearch.content.searchIndex.mappings.noIndex', { + defaultMessage: + "An Access Control Index won't be created until you enable document-level security and run your first access control sync.", + })} +

+ + ) : ( + + {JSON.stringify(mappingData, null, 2)} + + )} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.tsx index c26dce2885b0a..acd1db66413d1 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { useValues } from 'kea'; @@ -21,6 +21,8 @@ import { Chat } from '@kbn/cloud-chat-plugin/public'; import { i18n } from '@kbn/i18n'; import { WelcomeBanner } from '@kbn/search-api-panels'; +import { AuthenticatedUser } from '@kbn/security-plugin/common'; + import { ErrorStateCallout } from '../../../shared/error_state'; import { HttpLogic } from '../../../shared/http'; import { KibanaLogic } from '../../../shared/kibana'; @@ -40,9 +42,24 @@ import { IngestionSelector } from './ingestion_selector'; import './product_selector.scss'; export const ProductSelector: React.FC = () => { - const { config, userProfile } = useValues(KibanaLogic); + const { config, security } = useValues(KibanaLogic); const { errorConnectingMessage } = useValues(HttpLogic); + const [user, setUser] = useState({}); + + useEffect(() => { + try { + security.authc + .getCurrentUser() + .then(setUser) + .catch(() => { + setUser({}); + }); + } catch { + setUser({}); + } + }, [security.authc]); + const showErrorConnecting = !!(config.host && errorConnectingMessage); // The create index flow does not work without ent-search, when content is updated // to no longer rely on ent-search we can always show the Add Content component @@ -53,7 +70,7 @@ export const ProductSelector: React.FC = () => { - + diff --git a/x-pack/plugins/enterprise_search/public/applications/index.tsx b/x-pack/plugins/enterprise_search/public/applications/index.tsx index 0cee920d4ff7f..def858dfb6040 100644 --- a/x-pack/plugins/enterprise_search/public/applications/index.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/index.tsx @@ -66,7 +66,7 @@ export const renderApp = ( const { history } = params; const { application, chrome, http, uiSettings } = core; const { capabilities, navigateToUrl } = application; - const { charts, cloud, guidedOnboarding, lens, security, share, userProfile } = plugins; + const { charts, cloud, guidedOnboarding, lens, security, share } = plugins; const entCloudHost = getCloudEnterpriseSearchHost(plugins.cloud); externalUrl.enterpriseSearchUrl = publicUrl || entCloudHost || config.host || ''; @@ -108,7 +108,6 @@ export const renderApp = ( setDocTitle: chrome.docTitle.change, share, uiSettings, - userProfile, }); const unmountLicensingLogic = mountLicensingLogic({ canManageLicense: core.application.capabilities.management?.stack?.license_management, diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/kibana/kibana_logic.ts b/x-pack/plugins/enterprise_search/public/applications/shared/kibana/kibana_logic.ts index c79ad565b2eb7..d816c747e5027 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/kibana/kibana_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/kibana/kibana_logic.ts @@ -21,7 +21,6 @@ import { import { DataPublicPluginStart } from '@kbn/data-plugin/public'; import { GuidedOnboardingPluginStart } from '@kbn/guided-onboarding-plugin/public'; import { LensPublicStart } from '@kbn/lens-plugin/public'; -import { GetUserProfileResponse, UserProfileData } from '@kbn/security-plugin/common'; import { SecurityPluginStart } from '@kbn/security-plugin/public'; import { SharePluginStart } from '@kbn/share-plugin/public'; @@ -54,7 +53,6 @@ interface KibanaLogicProps { setDocTitle(title: string): void; share: SharePluginStart; uiSettings: IUiSettingsClient; - userProfile: GetUserProfileResponse; } export interface KibanaValues extends Omit { @@ -95,7 +93,6 @@ export const KibanaLogic = kea>({ setDocTitle: [props.setDocTitle, {}], share: [props.share, {}], uiSettings: [props.uiSettings, {}], - userProfile: [props.userProfile, {}], }), selectors: ({ selectors }) => ({ isCloud: [() => [selectors.cloud], (cloud?: Partial) => !!cloud?.isCloudEnabled], diff --git a/x-pack/plugins/enterprise_search/public/plugin.ts b/x-pack/plugins/enterprise_search/public/plugin.ts index e0b86110125fb..615805190a3bf 100644 --- a/x-pack/plugins/enterprise_search/public/plugin.ts +++ b/x-pack/plugins/enterprise_search/public/plugin.ts @@ -22,7 +22,6 @@ import { GuidedOnboardingPluginStart } from '@kbn/guided-onboarding-plugin/publi import type { HomePublicPluginSetup } from '@kbn/home-plugin/public'; import { LensPublicStart } from '@kbn/lens-plugin/public'; import { LicensingPluginStart } from '@kbn/licensing-plugin/public'; -import { GetUserProfileResponse, UserProfileData } from '@kbn/security-plugin/common'; import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/public'; import { SharePluginStart } from '@kbn/share-plugin/public'; @@ -66,7 +65,6 @@ export interface PluginsStart { licensing: LicensingPluginStart; security: SecurityPluginStart; share: SharePluginStart; - userProfile: GetUserProfileResponse; } export class EnterpriseSearchPlugin implements Plugin { @@ -102,8 +100,7 @@ export class EnterpriseSearchPlugin implements Plugin { cloudSetup && (pluginsStart as PluginsStart).cloud ? { ...cloudSetup, ...(pluginsStart as PluginsStart).cloud } : undefined; - const userProfile = await (pluginsStart as PluginsStart).security.userProfiles.getCurrent(); - const plugins = { ...pluginsStart, cloud, userProfile } as PluginsStart; + const plugins = { ...pluginsStart, cloud } as PluginsStart; coreStart.chrome .getChromeStyle$() diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts index 52cb24271afa5..3548fee93fbf2 100644 --- a/x-pack/plugins/fleet/common/constants/epm.ts +++ b/x-pack/plugins/fleet/common/constants/epm.ts @@ -9,7 +9,7 @@ import { ElasticsearchAssetType, KibanaAssetType } from '../types/models'; export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages'; export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets'; -export const MAX_TIME_COMPLETE_INSTALL = 60000; +export const MAX_TIME_COMPLETE_INSTALL = 30 * 60 * 1000; // 30 minutes export const FLEET_SYSTEM_PACKAGE = 'system'; export const FLEET_ELASTIC_AGENT_PACKAGE = 'elastic_agent'; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx index 014075b1f0241..e949f6ec64fd9 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx @@ -204,7 +204,7 @@ export const AgentUpgradeAgentModal: React.FunctionComponent { - const normalizedSearchValue = searchValue.trim().toLowerCase(); + const normalizedSearchValue = searchValue.trim(); const newOption = { label: normalizedSearchValue, diff --git a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/kubernetes_instructions.test.tsx b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/kubernetes_instructions.test.tsx new file mode 100644 index 0000000000000..9eab4d3e0c99a --- /dev/null +++ b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/kubernetes_instructions.test.tsx @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getManifestDownloadLink } from './kubernetes_instructions'; + +describe('getManifestDownloadLink', () => { + it('should return the correct link', () => { + expect(getManifestDownloadLink('https://fleet.host', 'enrollmentToken')).toEqual( + '/api/fleet/kubernetes/download?fleetServer=https%3A%2F%2Ffleet.host&enrolToken=enrollmentToken' + ); + expect(getManifestDownloadLink('https://fleet.host')).toEqual( + '/api/fleet/kubernetes/download?fleetServer=https%3A%2F%2Ffleet.host' + ); + expect(getManifestDownloadLink(undefined, 'enrollmentToken')).toEqual( + '/api/fleet/kubernetes/download?enrolToken=enrollmentToken' + ); + }); +}); diff --git a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/kubernetes_instructions.tsx b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/kubernetes_instructions.tsx index 8125cdb4acf35..a44b7ab4020e9 100644 --- a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/kubernetes_instructions.tsx +++ b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/kubernetes_instructions.tsx @@ -31,6 +31,15 @@ interface Props { fleetServerHost?: string; } +export const getManifestDownloadLink = (fleetServerHost?: string, enrollmentAPIKey?: string) => { + const searchParams = new URLSearchParams({ + ...(fleetServerHost && { fleetServer: fleetServerHost }), + ...(enrollmentAPIKey && { enrolToken: enrollmentAPIKey }), + }); + + return `${agentPolicyRouteService.getK8sFullDownloadPath()}?${searchParams.toString()}`; +}; + export const KubernetesInstructions: React.FunctionComponent = ({ enrollmentAPIKey, onCopy, @@ -111,13 +120,8 @@ export const KubernetesInstructions: React.FunctionComponent = ({ ); - const searchParams = new URLSearchParams({ - ...(fleetServerHost && { fleetServer: fleetServerHost }), - ...(enrollmentAPIKey && { enrolToken: enrollmentAPIKey }), - }); - const downloadLink = core.http.basePath.prepend( - `${agentPolicyRouteService.getK8sFullDownloadPath()}${searchParams.toString()}` + getManifestDownloadLink(fleetServerHost, enrollmentAPIKey) ); const k8sDownloadYaml = ( diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 7949c9ab98a71..5682749d7e381 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -11,6 +11,8 @@ import type { MappingTypeMapping, } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import pMap from 'p-map'; + import type { Field, Fields } from '../../fields/field'; import type { RegistryDataStream, @@ -534,9 +536,15 @@ export function generateTemplateName(dataStream: RegistryDataStream): string { /** * Given a data stream name, return the indexTemplate name */ -function dataStreamNameToIndexTemplateName(dataStreamName: string): string { - const [type, dataset] = dataStreamName.split('-'); // ignore namespace at the end - return [type, dataset].join('-'); +async function getIndexTemplate( + esClient: ElasticsearchClient, + dataStreamName: string +): Promise { + const dataStream = await esClient.indices.getDataStream({ + name: dataStreamName, + expand_wildcards: ['open', 'hidden'], + }); + return dataStream.data_streams[0].template; } export function generateTemplateIndexPattern(dataStream: RegistryDataStream): string { @@ -723,15 +731,22 @@ const updateAllDataStreams = async ( esClient: ElasticsearchClient, logger: Logger ): Promise => { - const updatedataStreamPromises = indexNameWithTemplates.map((templateEntry) => { - return updateExistingDataStream({ - esClient, - logger, - dataStreamName: templateEntry.dataStreamName, - }); - }); - await Promise.all(updatedataStreamPromises); + await pMap( + indexNameWithTemplates, + (templateEntry) => { + return updateExistingDataStream({ + esClient, + logger, + dataStreamName: templateEntry.dataStreamName, + }); + }, + { + // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster + concurrency: 20, + } + ); }; + const updateExistingDataStream = async ({ dataStreamName, esClient, @@ -757,9 +772,9 @@ const updateExistingDataStream = async ({ let lifecycle: any; try { - const simulateResult = await retryTransientEsErrors(() => + const simulateResult = await retryTransientEsErrors(async () => esClient.indices.simulateTemplate({ - name: dataStreamNameToIndexTemplateName(dataStreamName), + name: await getIndexTemplate(esClient, dataStreamName), }) ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts index 96582ff554190..850164f51787d 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts @@ -5,11 +5,21 @@ * 2.0. */ -import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { loggerMock } from '@kbn/logging-mocks'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; +import { ConcurrentInstallOperationError } from '../../../errors'; + +import type { Installation } from '../../../../common'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../common'; + import { appContextService } from '../../app_context'; import { createAppContextStartContractMock } from '../../../mocks'; import { saveArchiveEntries } from '../archive/storage'; @@ -29,7 +39,9 @@ jest.mock('../elasticsearch/datastream_ilm/install'); import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; -import { installIndexTemplatesAndPipelines } from './install'; +import { MAX_TIME_COMPLETE_INSTALL } from '../../../../common/constants'; + +import { installIndexTemplatesAndPipelines, restartInstallation } from './install'; import { _installPackage } from './_install_package'; @@ -69,9 +81,7 @@ describe('_installPackage', () => { jest.mocked(saveArchiveEntries).mockResolvedValue({ saved_objects: [], }); - }); - afterEach(async () => { - appContextService.stop(); + jest.mocked(restartInstallation).mockReset(); }); it('handles errors from installKibanaAssets', async () => { // force errors from this function @@ -220,4 +230,125 @@ describe('_installPackage', () => { expect(installILMPolicy).toBeCalled(); expect(installIlmForDataStream).toBeCalled(); }); + + describe('when package is stuck in `installing`', () => { + afterEach(() => {}); + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + + beforeEach(() => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + disableProxies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + capabilities: [], + }, + }) + ); + }); + + describe('timeout reached', () => { + it('restarts installation', async () => { + await _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date( + Date.now() - MAX_TIME_COMPLETE_INSTALL * 2 + ).toISOString(), + }, + }, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + + describe('timeout not reached', () => { + describe('force flag not provided', () => { + it('throws concurrent installation error if force flag is not provided', async () => { + expect( + _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + }) + ).rejects.toThrowError(ConcurrentInstallOperationError); + }); + }); + + describe('force flag provided', () => { + it('restarts installation', async () => { + await _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + force: true, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 337cf59bbd613..3bfc74bf68968 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -99,18 +99,30 @@ export async function _installPackage({ try { // if some installation already exists if (installedPkg) { + const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; + const hasExceededTimeout = + Date.now() - Date.parse(installedPkg.attributes.install_started_at) < + MAX_TIME_COMPLETE_INSTALL; + // if the installation is currently running, don't try to install // instead, only return already installed assets - if ( - installedPkg.attributes.install_status === 'installing' && - Date.now() - Date.parse(installedPkg.attributes.install_started_at) < - MAX_TIME_COMPLETE_INSTALL - ) { - throw new ConcurrentInstallOperationError( - `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ - pkgVersion || 'unknown' - } detected, aborting.` - ); + if (isStatusInstalling && hasExceededTimeout) { + // If this is a forced installation, ignore the timeout and restart the installation anyway + if (force) { + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } else { + throw new ConcurrentInstallOperationError( + `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + } detected, aborting.` + ); + } } else { // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL // (it might be stuck) update the saved object and proceed diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap index 5f828ee51b625..aea9ba75f2863 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap @@ -24,7 +24,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -130,7 +130,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -163,7 +163,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -277,7 +277,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -391,7 +391,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -505,7 +505,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -765,7 +765,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -1039,7 +1039,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap index dc9278511a864..b7c4ebf0f6b8c 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap @@ -20,7 +20,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -122,7 +122,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -151,7 +151,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -261,7 +261,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -371,7 +371,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -481,7 +481,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -730,7 +730,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -993,7 +993,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap index ee4001303fec5..ad100c4662fe9 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap @@ -51,7 +51,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -232,7 +232,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -488,7 +488,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap index 126437173f84a..3297a8c513821 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap @@ -92,7 +92,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -247,7 +247,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -477,7 +477,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap index ced0801d859d4..362b1d4a9e01e 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap @@ -63,7 +63,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -256,7 +256,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -524,7 +524,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts index 3dd6469a7f2c5..171a5ca15e2de 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts @@ -72,7 +72,7 @@ export class ApmTransactionDurationTransformGenerator extends TransformGenerator { range: { '@timestamp': { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts index 3818111c70df2..d038876f5ee9d 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts @@ -72,7 +72,7 @@ export class ApmTransactionErrorRateTransformGenerator extends TransformGenerato { range: { '@timestamp': { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts index 654f8d67a3673..844703003257a 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts @@ -54,7 +54,7 @@ export class HistogramTransformGenerator extends TransformGenerator { { range: { [indicator.params.timestampField]: { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts index 8321a0cb7172e..02c7757ec1362 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts @@ -49,7 +49,7 @@ export class KQLCustomTransformGenerator extends TransformGenerator { { range: { [indicator.params.timestampField]: { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts index 52209533f828c..063e6fbe1e3dc 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts @@ -52,7 +52,7 @@ export class MetricCustomTransformGenerator extends TransformGenerator { { range: { [indicator.params.timestampField]: { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts b/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts index c734af44b36f4..93b30775170dc 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts @@ -23,7 +23,7 @@ export default function (providerContext: FtrProviderContext) { skipIfNoDockerRegistry(providerContext); setupFleetAndAgents(providerContext); - after(async () => { + afterEach(async () => { await deletePackage('apm', '8.8.0'); }); @@ -88,5 +88,60 @@ export default function (providerContext: FtrProviderContext) { // datastream rolled over expect(Object.keys(ds).length).greaterThan(1); }); + + it('should not rollover datastreams when successfully updated mappings', async function () { + await supertest + .post(`/api/fleet/epm/packages/apm/8.8.0`) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + + await es.index({ + index: 'metrics-apm.app.default-default', + document: { + '@timestamp': '2023-05-30T07:50:00.000Z', + agent: { + name: 'go', + }, + data_stream: { + dataset: 'metrics-apm.app.default', + namespace: 'default', + type: 'metrics', + }, + ecs: { + version: '8.8.0-dev', + }, + event: { + agent_id_status: 'missing', + ingested: '2023-05-30T07:57:12Z', + }, + observer: { + hostname: '047e282994fb', + type: 'apm-server', + version: '8.8.0', + }, + }, + }); + + let ds = await es.indices.get({ + index: 'metrics-apm.app.default*', + expand_wildcards: ['open', 'hidden'], + }); + const indicesBefore = Object.keys(ds).length; + + await supertest + .post(`/api/fleet/epm/packages/apm/8.8.0`) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + + ds = await es.indices.get({ + index: 'metrics-apm.app.default*', + expand_wildcards: ['open', 'hidden'], + }); + const indicesAfter = Object.keys(ds).length; + // datastream did not roll over + expect(indicesAfter).equal(indicesBefore); + }); }); } diff --git a/yarn.lock b/yarn.lock index ae34a37e24657..ec64a7f01b794 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15988,7 +15988,7 @@ exec-sh@^0.3.2: resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.2.tgz#6738de2eb7c8e671d0366aea0b0db8c6f7d7391b" integrity sha512-9sLAvzhI5nc8TpuQUh4ahMdCrWT00wPWz7j47/emR5+2qEfoZP5zzUXvx+vdx+H6ohhnsYC31iX04QLYJK8zTg== -execa@4.1.0, execa@^4.0.2: +execa@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==