From da8264f26a08abf47a50035549f8cccb30fc5875 Mon Sep 17 00:00:00 2001 From: Zacqary Adam Xeper Date: Wed, 13 Oct 2021 10:36:07 -0500 Subject: [PATCH 01/35] [Stack Monitoring] Fix hashchange detection on sidenav link (#114727) --- x-pack/plugins/monitoring/public/application/index.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/monitoring/public/application/index.tsx b/x-pack/plugins/monitoring/public/application/index.tsx index 7b4c73475338f..2caead4f963a1 100644 --- a/x-pack/plugins/monitoring/public/application/index.tsx +++ b/x-pack/plugins/monitoring/public/application/index.tsx @@ -59,9 +59,15 @@ import { LogStashNodePipelinesPage } from './pages/logstash/node_pipelines'; export const renderApp = ( core: CoreStart, plugins: MonitoringStartPluginDependencies, - { element, setHeaderActionMenu }: AppMountParameters, + { element, history, setHeaderActionMenu }: AppMountParameters, externalConfig: ExternalConfig ) => { + // dispatch synthetic hash change event to update hash history objects + // this is necessary because hash updates triggered by using popState won't trigger this event naturally. + const unlistenParentHistory = history.listen(() => { + window.dispatchEvent(new HashChangeEvent('hashchange')); + }); + ReactDOM.render( { ReactDOM.unmountComponentAtNode(element); + unlistenParentHistory(); }; }; From 6c5dd08712ac741a45a2732bf53084daae783abe Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 13 Oct 2021 10:45:27 -0500 Subject: [PATCH 02/35] [ts] Check .d.ts files for all projects in typeCheck (#114295) * enable --skip-lib-check for all projects in typeCheck and fix existing issues * fix graph types * transpile TS to ES2019, but not all the way back to es5 Co-authored-by: spalger Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- src/dev/typescript/run_type_check_cli.ts | 19 ++----------------- .../workspace/graph_client_workspace.d.ts | 2 +- x-pack/plugins/reporting/server/lib/puid.d.ts | 2 +- .../security_solution/cypress/tsconfig.json | 7 +++---- 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/src/dev/typescript/run_type_check_cli.ts b/src/dev/typescript/run_type_check_cli.ts index 472b9c04757ca..27998f881a03d 100644 --- a/src/dev/typescript/run_type_check_cli.ts +++ b/src/dev/typescript/run_type_check_cli.ts @@ -53,23 +53,8 @@ export async function runTypeCheckCli() { } } - const nonCompositeProjects = projects.filter((p) => !p.isCompositeProject()); - if (!nonCompositeProjects.length) { - if (projectFilter) { - log.success( - `${flags.project} is a composite project so its types are validated by scripts/build_ts_refs` - ); - } else { - log.success( - `All projects are composite so their types are validated by scripts/build_ts_refs` - ); - } - - return; - } - const concurrency = Math.min(4, Math.round((Os.cpus() || []).length / 2) || 1) || 1; - log.info('running type check in', nonCompositeProjects.length, 'non-composite projects'); + log.info('running type check in', projects.length, 'projects'); const tscArgs = [ ...['--emitDeclarationOnly', 'false'], @@ -81,7 +66,7 @@ export async function runTypeCheckCli() { ]; const failureCount = await lastValueFrom( - Rx.from(nonCompositeProjects).pipe( + Rx.from(projects).pipe( mergeMap(async (p) => { const relativePath = Path.relative(process.cwd(), p.tsConfigPath); diff --git a/x-pack/plugins/graph/public/services/workspace/graph_client_workspace.d.ts b/x-pack/plugins/graph/public/services/workspace/graph_client_workspace.d.ts index b369ec6c5762c..18d08d8da31d8 100644 --- a/x-pack/plugins/graph/public/services/workspace/graph_client_workspace.d.ts +++ b/x-pack/plugins/graph/public/services/workspace/graph_client_workspace.d.ts @@ -5,6 +5,6 @@ * 2.0. */ -import { Workspace, WorkspaceOptions } from '../types'; +import { Workspace, WorkspaceOptions } from '../../types'; declare function createWorkspace(options: WorkspaceOptions): Workspace; diff --git a/x-pack/plugins/reporting/server/lib/puid.d.ts b/x-pack/plugins/reporting/server/lib/puid.d.ts index 2cf7dad67d06e..4ac240157971f 100644 --- a/x-pack/plugins/reporting/server/lib/puid.d.ts +++ b/x-pack/plugins/reporting/server/lib/puid.d.ts @@ -6,7 +6,7 @@ */ declare module 'puid' { - declare class Puid { + class Puid { generate(): string; } diff --git a/x-pack/plugins/security_solution/cypress/tsconfig.json b/x-pack/plugins/security_solution/cypress/tsconfig.json index 4efb4c5c56296..a779c3f48d346 100644 --- a/x-pack/plugins/security_solution/cypress/tsconfig.json +++ b/x-pack/plugins/security_solution/cypress/tsconfig.json @@ -8,15 +8,14 @@ "target/**/*" ], "compilerOptions": { - "target": "es5", - "lib": ["es5", "dom"], + "target": "ES2019", "outDir": "target/types", "types": [ "cypress", "cypress-pipe", - "node" + "node", + "resize-observer-polyfill", ], - "resolveJsonModule": true, }, "references": [ { "path": "../tsconfig.json" } From 330fd83aaf8750fc090722a3cbbb011cf3574be5 Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Wed, 13 Oct 2021 12:00:21 -0400 Subject: [PATCH 03/35] [Security Solution][Endpoint] Policy Details Trusted Apps tab Remove action for single card (#114207) * Remove modal for removing a policy from card * new `assignPolicyToTrustedApps()` and `removePolicyFromTrustedApps()` methods for TrustedAppsService * Additional tests for the policy details Trusted Apps List page/tab * several tests for RemoveTrustedAppFromPolicyModal (but not all) --- .../management/pages/mocks/fleet_mocks.ts | 5 +- .../action/policy_trusted_apps_action.ts | 20 ++- .../policy_trusted_apps_middleware.ts | 131 ++++++++------- .../reducer/initial_policy_details_state.ts | 1 + .../reducer/trusted_apps_reducer.ts | 20 +++ .../selectors/trusted_apps_selectors.ts | 21 +++ .../pages/policy/test_utils/mocks.ts | 3 +- .../public/management/pages/policy/types.ts | 9 + .../pages/policy/view/policy_hooks.ts | 25 ++- .../policy_trusted_apps_flyout.test.tsx | 5 + .../flyout/policy_trusted_apps_flyout.tsx | 14 +- .../list/policy_trusted_apps_list.test.tsx | 107 ++++++++---- .../list/policy_trusted_apps_list.tsx | 50 +++++- ...ove_trusted_app_from_policy_modal.test.tsx | 156 ++++++++++++++++++ .../remove_trusted_app_from_policy_modal.tsx | 155 +++++++++++++++++ .../pages/trusted_apps/service/index.ts | 97 ++++++++++- .../trusted_apps/store/middleware.test.ts | 2 + 17 files changed, 710 insertions(+), 111 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/remove_trusted_app_from_policy_modal.test.tsx create mode 100644 x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/remove_trusted_app_from_policy_modal.tsx diff --git a/x-pack/plugins/security_solution/public/management/pages/mocks/fleet_mocks.ts b/x-pack/plugins/security_solution/public/management/pages/mocks/fleet_mocks.ts index c9a972ce29e4c..8f1530c3632dc 100644 --- a/x-pack/plugins/security_solution/public/management/pages/mocks/fleet_mocks.ts +++ b/x-pack/plugins/security_solution/public/management/pages/mocks/fleet_mocks.ts @@ -52,9 +52,10 @@ export const fleetGetEndpointPackagePolicyHttpMock = path: PACKAGE_POLICY_API_ROUTES.INFO_PATTERN, method: 'get', handler: () => { - return { - items: new EndpointDocGenerator('seed').generatePolicyPackagePolicy(), + const response: GetPolicyResponse = { + item: new EndpointDocGenerator('seed').generatePolicyPackagePolicy(), }; + return response; }, }, ]); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/action/policy_trusted_apps_action.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/action/policy_trusted_apps_action.ts index 479452968df7a..b3bdfe32ef091 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/action/policy_trusted_apps_action.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/action/policy_trusted_apps_action.ts @@ -8,8 +8,10 @@ import { Action } from 'redux'; import { AsyncResourceState } from '../../../../../state'; import { - PostTrustedAppCreateResponse, + PutTrustedAppUpdateResponse, GetTrustedAppsListResponse, + TrustedApp, + MaybeImmutable, } from '../../../../../../../common/endpoint/types'; import { PolicyArtifactsState } from '../../../types'; @@ -21,13 +23,14 @@ export interface PolicyArtifactsAssignableListPageDataChanged { export interface PolicyArtifactsUpdateTrustedApps { type: 'policyArtifactsUpdateTrustedApps'; payload: { - trustedAppIds: string[]; + action: 'assign' | 'remove'; + artifacts: MaybeImmutable; }; } export interface PolicyArtifactsUpdateTrustedAppsChanged { type: 'policyArtifactsUpdateTrustedAppsChanged'; - payload: AsyncResourceState; + payload: AsyncResourceState; } export interface PolicyArtifactsAssignableListExistDataChanged { @@ -58,6 +61,13 @@ export interface PolicyDetailsListOfAllPoliciesStateChanged export type PolicyDetailsTrustedAppsForceListDataRefresh = Action<'policyDetailsTrustedAppsForceListDataRefresh'>; +export type PolicyDetailsArtifactsResetRemove = Action<'policyDetailsArtifactsResetRemove'>; + +export interface PolicyDetailsTrustedAppsRemoveListStateChanged + extends Action<'policyDetailsTrustedAppsRemoveListStateChanged'> { + payload: PolicyArtifactsState['removeList']; +} + /** * All of the possible actions for Trusted Apps under the Policy Details store */ @@ -70,4 +80,6 @@ export type PolicyTrustedAppsAction = | PolicyArtifactsDeosAnyTrustedAppExists | AssignedTrustedAppsListStateChanged | PolicyDetailsListOfAllPoliciesStateChanged - | PolicyDetailsTrustedAppsForceListDataRefresh; + | PolicyDetailsTrustedAppsForceListDataRefresh + | PolicyDetailsTrustedAppsRemoveListStateChanged + | PolicyDetailsArtifactsResetRemove; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware/policy_trusted_apps_middleware.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware/policy_trusted_apps_middleware.ts index 360fe9fb99b8d..f50eb342acba1 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware/policy_trusted_apps_middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware/policy_trusted_apps_middleware.ts @@ -5,43 +5,44 @@ * 2.0. */ -import pMap from 'p-map'; -import { find, isEmpty } from 'lodash/fp'; +import { isEmpty } from 'lodash/fp'; import { - PolicyDetailsState, - MiddlewareRunner, GetPolicyListResponse, + MiddlewareRunner, MiddlewareRunnerContext, PolicyAssignedTrustedApps, + PolicyDetailsState, PolicyDetailsStore, + PolicyRemoveTrustedApps, } from '../../../types'; import { - policyIdFromParams, - getAssignableArtifactsList, doesPolicyTrustedAppsListNeedUpdate, + getCurrentArtifactsLocation, getCurrentPolicyAssignedTrustedAppsState, + getCurrentTrustedAppsRemoveListState, + getCurrentUrlLocationPaginationParams, getLatestLoadedPolicyAssignedTrustedAppsState, + getTrustedAppsIsRemoving, getTrustedAppsPolicyListState, - isPolicyTrustedAppListLoading, - getCurrentArtifactsLocation, isOnPolicyTrustedAppsView, - getCurrentUrlLocationPaginationParams, + isPolicyTrustedAppListLoading, + licensedPolicy, + policyIdFromParams, getDoesAnyTrustedAppExistsIsLoading, } from '../selectors'; import { - ImmutableArray, - ImmutableObject, - PostTrustedAppCreateRequest, - TrustedApp, Immutable, + MaybeImmutable, + PutTrustedAppUpdateResponse, + TrustedApp, } from '../../../../../../../common/endpoint/types'; import { ImmutableMiddlewareAPI } from '../../../../../../common/store'; import { TrustedAppsService } from '../../../../trusted_apps/service'; import { + createFailedResourceState, createLoadedResourceState, createLoadingResourceState, createUninitialisedResourceState, - createFailedResourceState, isLoadingResourceState, isUninitialisedResourceState, } from '../../../../../state'; @@ -83,8 +84,13 @@ export const policyTrustedAppsMiddlewareRunner: MiddlewareRunner = async ( break; case 'policyArtifactsUpdateTrustedApps': - if (getCurrentArtifactsLocation(state).show === 'list') { - await updateTrustedApps(store, trustedAppsService, action.payload.trustedAppIds); + if ( + getCurrentArtifactsLocation(state).show === 'list' && + action.payload.action === 'assign' + ) { + await updateTrustedApps(store, trustedAppsService, action.payload.artifacts); + } else if (action.payload.action === 'remove') { + removeTrustedAppsFromPolicy(context, store, action.payload.artifacts); } break; @@ -213,58 +219,26 @@ const searchTrustedApps = async ( } }; -interface UpdateTrustedAppWrapperProps { - entry: ImmutableObject; - policies: ImmutableArray; -} - const updateTrustedApps = async ( store: ImmutableMiddlewareAPI, trustedAppsService: TrustedAppsService, - trustedAppsIds: ImmutableArray + trustedApps: MaybeImmutable ) => { const state = store.getState(); const policyId = policyIdFromParams(state); - const availavleArtifacts = getAssignableArtifactsList(state); - - if (!availavleArtifacts || !availavleArtifacts.data.length) { - return; - } store.dispatch({ type: 'policyArtifactsUpdateTrustedAppsChanged', // Ignore will be fixed with when AsyncResourceState is refactored (#830) - // @ts-ignore + // @ts-expect-error payload: createLoadingResourceState({ previousState: createUninitialisedResourceState() }), }); try { - const trustedAppsUpdateActions = []; - - const updateTrustedApp = async ({ entry, policies }: UpdateTrustedAppWrapperProps) => - trustedAppsService.updateTrustedApp({ id: entry.id }, { - effectScope: { type: 'policy', policies: [...policies, policyId] }, - name: entry.name, - entries: entry.entries, - os: entry.os, - description: entry.description, - version: entry.version, - } as PostTrustedAppCreateRequest); - - for (const entryId of trustedAppsIds) { - const entry = find({ id: entryId }, availavleArtifacts.data) as ImmutableObject; - if (entry) { - const policies = entry.effectScope.type === 'policy' ? entry.effectScope.policies : []; - trustedAppsUpdateActions.push({ entry, policies }); - } - } - - const updatedTrustedApps = await pMap(trustedAppsUpdateActions, updateTrustedApp, { - concurrency: 5, - /** When set to false, instead of stopping when a promise rejects, it will wait for all the promises to settle - * and then reject with an aggregated error containing all the errors from the rejected promises. */ - stopOnError: false, - }); + const updatedTrustedApps = await trustedAppsService.assignPolicyToTrustedApps( + policyId, + trustedApps + ); store.dispatch({ type: 'policyArtifactsUpdateTrustedAppsChanged', @@ -275,9 +249,7 @@ const updateTrustedApps = async ( } catch (err) { store.dispatch({ type: 'policyArtifactsUpdateTrustedAppsChanged', - // Ignore will be fixed with when AsyncResourceState is refactored (#830) - // @ts-ignore - payload: createFailedResourceState(err.body ?? err), + payload: createFailedResourceState(err.body ?? err), }); } }; @@ -371,3 +343,48 @@ const fetchAllPoliciesIfNeeded = async ( }); } }; + +const removeTrustedAppsFromPolicy = async ( + { trustedAppsService }: MiddlewareRunnerContext, + { getState, dispatch }: PolicyDetailsStore, + trustedApps: MaybeImmutable +): Promise => { + const state = getState(); + + if (getTrustedAppsIsRemoving(state)) { + return; + } + + dispatch({ + type: 'policyDetailsTrustedAppsRemoveListStateChanged', + // @ts-expect-error will be fixed when AsyncResourceState is refactored (#830) + payload: createLoadingResourceState(getCurrentTrustedAppsRemoveListState(state)), + }); + + try { + const currentPolicyId = licensedPolicy(state)?.id; + + if (!currentPolicyId) { + throw new Error('current policy id not found'); + } + + const response = await trustedAppsService.removePolicyFromTrustedApps( + currentPolicyId, + trustedApps + ); + + dispatch({ + type: 'policyDetailsTrustedAppsRemoveListStateChanged', + payload: createLoadedResourceState({ artifacts: trustedApps, response }), + }); + + dispatch({ + type: 'policyDetailsTrustedAppsForceListDataRefresh', + }); + } catch (error) { + dispatch({ + type: 'policyDetailsTrustedAppsRemoveListStateChanged', + payload: createFailedResourceState(error.body || error), + }); + } +}; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/reducer/initial_policy_details_state.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/reducer/initial_policy_details_state.ts index 2ad7ac9c06dac..008bcd262ceff 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/reducer/initial_policy_details_state.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/reducer/initial_policy_details_state.ts @@ -40,5 +40,6 @@ export const initialPolicyDetailsState: () => Immutable = () doesAnyTrustedAppExists: createUninitialisedResourceState(), assignedList: createUninitialisedResourceState(), policies: createUninitialisedResourceState(), + removeList: createUninitialisedResourceState(), }, }); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/reducer/trusted_apps_reducer.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/reducer/trusted_apps_reducer.ts index fbf498797e804..f9d090647b1b5 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/reducer/trusted_apps_reducer.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/reducer/trusted_apps_reducer.ts @@ -90,5 +90,25 @@ export const policyTrustedAppsReducer: ImmutableReducer = (state) => state.artifacts.removeList; + +export const getTrustedAppsIsRemoving: PolicyDetailsSelector = createSelector( + getCurrentTrustedAppsRemoveListState, + (removeListState) => isLoadingResourceState(removeListState) +); + +export const getTrustedAppsRemovalError: PolicyDetailsSelector = + createSelector(getCurrentTrustedAppsRemoveListState, (removeListState) => { + if (isFailedResourceState(removeListState)) { + return removeListState.error; + } + }); + +export const getTrustedAppsWasRemoveSuccessful: PolicyDetailsSelector = createSelector( + getCurrentTrustedAppsRemoveListState, + (removeListState) => isLoadedResourceState(removeListState) +); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/test_utils/mocks.ts b/x-pack/plugins/security_solution/public/management/pages/policy/test_utils/mocks.ts index be38e591dd9da..aca0971621863 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/test_utils/mocks.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/test_utils/mocks.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { HttpFetchOptionsWithPath } from 'kibana/public'; import { composeHttpHandlerMocks, httpHandlerMockFactory, @@ -71,7 +72,7 @@ export const getAPIError = () => ({ }); type PolicyDetailsTrustedAppsHttpMocksInterface = ResponseProvidersInterface<{ - policyTrustedAppsList: () => GetTrustedAppsListResponse; + policyTrustedAppsList: (options: HttpFetchOptionsWithPath) => GetTrustedAppsListResponse; }>; /** diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/types.ts b/x-pack/plugins/security_solution/public/management/pages/policy/types.ts index 7f8bf9c3872ea..283c3afb573b6 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/types.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/types.ts @@ -16,6 +16,8 @@ import { PostTrustedAppCreateResponse, MaybeImmutable, GetTrustedAppsListResponse, + TrustedApp, + PutTrustedAppUpdateResponse, } from '../../../../common/endpoint/types'; import { ServerApiError } from '../../../common/types'; import { @@ -78,6 +80,11 @@ export interface PolicyAssignedTrustedApps { artifacts: GetTrustedAppsListResponse; } +export interface PolicyRemoveTrustedApps { + artifacts: TrustedApp[]; + response: PutTrustedAppUpdateResponse[]; +} + /** * Policy artifacts store state */ @@ -96,6 +103,8 @@ export interface PolicyArtifactsState { assignedList: AsyncResourceState; /** A list of all available polices */ policies: AsyncResourceState; + /** list of artifacts to remove. Holds the ids that were removed and the API response */ + removeList: AsyncResourceState; } export enum OS { diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_hooks.ts b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_hooks.ts index 82296730af686..c6b89b4137cc4 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_hooks.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_hooks.ts @@ -71,6 +71,8 @@ export const usePolicyTrustedAppsNotification = () => { if (updateSuccessfull && updatedArtifacts && !wasAlreadyHandled.has(updatedArtifacts)) { wasAlreadyHandled.add(updatedArtifacts); + const updateCount = updatedArtifacts.length; + toasts.addSuccess({ title: i18n.translate( 'xpack.securitySolution.endpoint.policy.trustedApps.layout.flyout.toastSuccess.title', @@ -78,13 +80,22 @@ export const usePolicyTrustedAppsNotification = () => { defaultMessage: 'Success', } ), - text: i18n.translate( - 'xpack.securitySolution.endpoint.policy.trustedApps.layout.flyout.toastSuccess.text', - { - defaultMessage: '"{names}" has been added to your trusted applications list.', - values: { names: updatedArtifacts.map((artifact) => artifact.data.name).join(', ') }, - } - ), + text: + updateCount > 1 + ? i18n.translate( + 'xpack.securitySolution.endpoint.policy.trustedApps.layout.flyout.toastSuccess.textMultiples', + { + defaultMessage: '{count} trusted applications have been added to your list.', + values: { count: updateCount }, + } + ) + : i18n.translate( + 'xpack.securitySolution.endpoint.policy.trustedApps.layout.flyout.toastSuccess.textSingle', + { + defaultMessage: '"{name}" has been added to your trusted applications list.', + values: { name: updatedArtifacts[0]!.data.name }, + } + ), }); } else if (updateFailed) { toasts.addDanger( diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/flyout/policy_trusted_apps_flyout.test.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/flyout/policy_trusted_apps_flyout.test.tsx index b07005908ed1e..c1d00f7a3f99b 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/flyout/policy_trusted_apps_flyout.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/flyout/policy_trusted_apps_flyout.test.tsx @@ -38,6 +38,11 @@ describe('Policy trusted apps flyout', () => { updateTrustedApp: () => ({ data: getMockCreateResponse(), }), + assignPolicyToTrustedApps: () => [ + { + data: getMockCreateResponse(), + }, + ], }; }); mockedContext = createAppRootMockRenderer(); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/flyout/policy_trusted_apps_flyout.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/flyout/policy_trusted_apps_flyout.tsx index cd291ed9f6eb0..63c7d5375476c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/flyout/policy_trusted_apps_flyout.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/flyout/policy_trusted_apps_flyout.tsx @@ -25,6 +25,7 @@ import { EuiCallOut, EuiEmptyPrompt, } from '@elastic/eui'; +import { Dispatch } from 'redux'; import { policyDetails, getCurrentArtifactsLocation, @@ -42,10 +43,12 @@ import { } from '../../policy_hooks'; import { PolicyArtifactsAssignableList } from '../../artifacts/assignable'; import { SearchExceptions } from '../../../../../components/search_exceptions'; +import { AppAction } from '../../../../../../common/store/actions'; +import { MaybeImmutable, TrustedApp } from '../../../../../../../common/endpoint/types'; export const PolicyTrustedAppsFlyout = React.memo(() => { usePolicyTrustedAppsNotification(); - const dispatch = useDispatch(); + const dispatch = useDispatch>(); const [selectedArtifactIds, setSelectedArtifactIds] = useState([]); const location = usePolicyDetailsSelector(getCurrentArtifactsLocation); const policyItem = usePolicyDetailsSelector(policyDetails); @@ -85,9 +88,14 @@ export const PolicyTrustedAppsFlyout = React.memo(() => { const handleOnConfirmAction = useCallback(() => { dispatch({ type: 'policyArtifactsUpdateTrustedApps', - payload: { trustedAppIds: selectedArtifactIds }, + payload: { + action: 'assign', + artifacts: selectedArtifactIds.map>((selectedId) => { + return assignableArtifactsList?.data?.find((trustedApp) => trustedApp.id === selectedId)!; + }), + }, }); - }, [dispatch, selectedArtifactIds]); + }, [assignableArtifactsList?.data, dispatch, selectedArtifactIds]); const handleOnSearch = useCallback( (filter) => { diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/policy_trusted_apps_list.test.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/policy_trusted_apps_list.test.tsx index 07b62d13e8edc..83709c50b76fa 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/policy_trusted_apps_list.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/policy_trusted_apps_list.test.tsx @@ -13,11 +13,19 @@ import { getPolicyDetailsArtifactsListPath } from '../../../../../common/routing import { PolicyTrustedAppsList } from './policy_trusted_apps_list'; import React from 'react'; import { policyDetailsPageAllApiHttpMocks } from '../../../test_utils'; -import { isFailedResourceState, isLoadedResourceState } from '../../../../../state'; +import { + createLoadingResourceState, + createUninitialisedResourceState, + isFailedResourceState, + isLoadedResourceState, +} from '../../../../../state'; import { fireEvent, within, act, waitFor } from '@testing-library/react'; import { APP_ID } from '../../../../../../../common/constants'; describe('when rendering the PolicyTrustedAppsList', () => { + // The index (zero based) of the card created by the generator that is policy specific + const POLICY_SPECIFIC_CARD_INDEX = 2; + let appTestContext: AppContextTestRender; let renderResult: ReturnType; let render: (waitForLoadedState?: boolean) => Promise>; @@ -84,8 +92,19 @@ describe('when rendering the PolicyTrustedAppsList', () => { }; }); - // FIXME: implement this test once PR #113802 is merged - it.todo('should show loading spinner if checking to see if trusted apps exist'); + it('should show loading spinner if checking to see if trusted apps exist', async () => { + await render(); + act(() => { + appTestContext.store.dispatch({ + type: 'policyArtifactsDeosAnyTrustedAppExists', + // Ignore will be fixed with when AsyncResourceState is refactored (#830) + // @ts-ignore + payload: createLoadingResourceState({ previousState: createUninitialisedResourceState() }), + }); + }); + + expect(renderResult.getByTestId('policyTrustedAppsGrid-loading')).not.toBeNull(); + }); it('should show total number of of items being displayed', async () => { await render(); @@ -163,40 +182,68 @@ describe('when rendering the PolicyTrustedAppsList', () => { ); }); - it('should display policy names on assignment context menu', async () => { - const retrieveAllPolicies = waitForAction('policyDetailsListOfAllPoliciesStateChanged', { - validate({ payload }) { - return isLoadedResourceState(payload); - }, - }); + it('should show dialog when remove action is clicked', async () => { await render(); - await retrieveAllPolicies; + await toggleCardActionMenu(POLICY_SPECIFIC_CARD_INDEX); act(() => { - fireEvent.click( - within(getCardByIndexPosition(2)).getByTestId( - 'policyTrustedAppsGrid-card-header-effectScope-popupMenu-button' + fireEvent.click(renderResult.getByTestId('policyTrustedAppsGrid-removeAction')); + }); + + await waitFor(() => expect(renderResult.getByTestId('confirmModalBodyText'))); + }); + + describe('and artifact is policy specific', () => { + const renderAndClickOnEffectScopePopupButton = async () => { + const retrieveAllPolicies = waitForAction('policyDetailsListOfAllPoliciesStateChanged', { + validate({ payload }) { + return isLoadedResourceState(payload); + }, + }); + await render(); + await retrieveAllPolicies; + act(() => { + fireEvent.click( + within(getCardByIndexPosition(POLICY_SPECIFIC_CARD_INDEX)).getByTestId( + 'policyTrustedAppsGrid-card-header-effectScope-popupMenu-button' + ) + ); + }); + await waitFor(() => + expect( + renderResult.getByTestId( + 'policyTrustedAppsGrid-card-header-effectScope-popupMenu-popoverPanel' + ) ) ); - }); - await waitFor(() => + }; + + it('should display policy names on assignment context menu', async () => { + await renderAndClickOnEffectScopePopupButton(); + expect( - renderResult.getByTestId( - 'policyTrustedAppsGrid-card-header-effectScope-popupMenu-popoverPanel' - ) - ) - ); + renderResult.getByTestId('policyTrustedAppsGrid-card-header-effectScope-popupMenu-item-0') + .textContent + ).toEqual('Endpoint Policy 0'); + expect( + renderResult.getByTestId('policyTrustedAppsGrid-card-header-effectScope-popupMenu-item-1') + .textContent + ).toEqual('Endpoint Policy 1'); + }); - expect( - renderResult.getByTestId('policyTrustedAppsGrid-card-header-effectScope-popupMenu-item-0') - .textContent - ).toEqual('Endpoint Policy 0'); - expect( - renderResult.getByTestId('policyTrustedAppsGrid-card-header-effectScope-popupMenu-item-1') - .textContent - ).toEqual('Endpoint Policy 1'); - }); + it('should navigate to policy details when clicking policy on assignment context menu', async () => { + await renderAndClickOnEffectScopePopupButton(); + + act(() => { + fireEvent.click( + renderResult.getByTestId('policyTrustedAppsGrid-card-header-effectScope-popupMenu-item-0') + ); + }); - it.todo('should navigate to policy details when clicking policy on assignment context menu'); + expect(appTestContext.history.location.pathname).toEqual( + '/administration/policy/ddf6570b-9175-4a6d-b288-61a09771c647/settings' + ); + }); + }); it('should handle pagination changes', async () => { await render(); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/policy_trusted_apps_list.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/policy_trusted_apps_list.tsx index 6793bee9c3c01..cb29d0ff868ac 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/policy_trusted_apps_list.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/policy_trusted_apps_list.tsx @@ -37,6 +37,7 @@ import { APP_ID } from '../../../../../../../common/constants'; import { ContextMenuItemNavByRouterProps } from '../../../../../components/context_menu_with_router_support/context_menu_item_nav_by_router'; import { ArtifactEntryCollapsibleCardProps } from '../../../../../components/artifact_entry_card'; import { useTestIdGenerator } from '../../../../../components/hooks/use_test_id_generator'; +import { RemoveTrustedAppFromPolicyModal } from './remove_trusted_app_from_policy_modal'; const DATA_TEST_SUBJ = 'policyTrustedAppsGrid'; @@ -56,6 +57,8 @@ export const PolicyTrustedAppsList = memo(() => { const trustedAppsApiError = usePolicyDetailsSelector(getPolicyTrustedAppListError); const [isCardExpanded, setCardExpanded] = useState>({}); + const [trustedAppsForRemoval, setTrustedAppsForRemoval] = useState([]); + const [showRemovalModal, setShowRemovalModal] = useState(false); const handlePageChange = useCallback( ({ pageIndex, pageSize }) => { @@ -100,6 +103,7 @@ export const PolicyTrustedAppsList = memo(() => { const newCardProps = new Map(); for (const trustedApp of trustedAppItems) { + const isGlobal = trustedApp.effectScope.type === 'global'; const viewUrlPath = getTrustedAppsListPath({ id: trustedApp.id, show: 'edit' }); const assignedPoliciesMenuItems: ArtifactEntryCollapsibleCardProps['policies'] = trustedApp.effectScope.type === 'global' @@ -142,6 +146,29 @@ export const PolicyTrustedAppsList = memo(() => { navigateOptions: { path: viewUrlPath }, 'data-test-subj': getTestId('viewFullDetailsAction'), }, + { + icon: 'trash', + children: i18n.translate( + 'xpack.securitySolution.endpoint.policy.trustedApps.list.removeAction', + { defaultMessage: 'Remove from policy' } + ), + onClick: () => { + setTrustedAppsForRemoval([trustedApp]); + setShowRemovalModal(true); + }, + disabled: isGlobal, + toolTipContent: isGlobal + ? i18n.translate( + 'xpack.securitySolution.endpoint.policy.trustedApps.list.removeActionNotAllowed', + { + defaultMessage: + 'Globally applied trusted applications cannot be removed from policy.', + } + ) + : undefined, + toolTipPosition: 'top', + 'data-test-subj': getTestId('removeAction'), + }, ], policies: assignedPoliciesMenuItems, }; @@ -159,6 +186,15 @@ export const PolicyTrustedAppsList = memo(() => { [cardProps] ); + const handleRemoveModalClose = useCallback(() => { + setShowRemovalModal(false); + }, []); + + // Anytime a new set of data (trusted apps) is retrieved, reset the card expand state + useEffect(() => { + setCardExpanded({}); + }, [trustedAppItems]); + // if an error occurred while loading the data, show toast useEffect(() => { if (trustedAppsApiError) { @@ -170,18 +206,13 @@ export const PolicyTrustedAppsList = memo(() => { } }, [toasts, trustedAppsApiError]); - // Anytime a new set of data (trusted apps) is retrieved, reset the card expand state - useEffect(() => { - setCardExpanded({}); - }, [trustedAppItems]); - if (hasTrustedApps.loading || isTrustedAppExistsCheckLoading) { return ( ); @@ -205,6 +236,13 @@ export const PolicyTrustedAppsList = memo(() => { pagination={pagination as Pagination} data-test-subj={DATA_TEST_SUBJ} /> + + {showRemovalModal && ( + + )} ); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/remove_trusted_app_from_policy_modal.test.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/remove_trusted_app_from_policy_modal.test.tsx new file mode 100644 index 0000000000000..fc2926bc65d3a --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/remove_trusted_app_from_policy_modal.test.tsx @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + AppContextTestRender, + createAppRootMockRenderer, +} from '../../../../../../common/mock/endpoint'; +import { getPolicyDetailsArtifactsListPath } from '../../../../../common/routing'; +import { isLoadedResourceState } from '../../../../../state'; +import React from 'react'; +import { fireEvent, act } from '@testing-library/react'; +import { policyDetailsPageAllApiHttpMocks } from '../../../test_utils'; +import { + RemoveTrustedAppFromPolicyModal, + RemoveTrustedAppFromPolicyModalProps, +} from './remove_trusted_app_from_policy_modal'; +import { PolicyArtifactsUpdateTrustedApps } from '../../../store/policy_details/action/policy_trusted_apps_action'; +import { Immutable } from '../../../../../../../common/endpoint/types'; +import { HttpFetchOptionsWithPath } from 'kibana/public'; + +describe('When using the RemoveTrustedAppFromPolicyModal component', () => { + let appTestContext: AppContextTestRender; + let renderResult: ReturnType; + let render: (waitForLoadedState?: boolean) => Promise>; + let waitForAction: AppContextTestRender['middlewareSpy']['waitForAction']; + let mockedApis: ReturnType; + let onCloseHandler: jest.MockedFunction; + let trustedApps: RemoveTrustedAppFromPolicyModalProps['trustedApps']; + + beforeEach(() => { + appTestContext = createAppRootMockRenderer(); + waitForAction = appTestContext.middlewareSpy.waitForAction; + onCloseHandler = jest.fn(); + mockedApis = policyDetailsPageAllApiHttpMocks(appTestContext.coreStart.http); + trustedApps = [ + mockedApis.responseProvider.policyTrustedAppsList({ query: {} } as HttpFetchOptionsWithPath) + .data[0], + ]; + + render = async (waitForLoadedState: boolean = true) => { + const pendingDataLoadState = waitForLoadedState + ? Promise.all([ + waitForAction('serverReturnedPolicyDetailsData'), + waitForAction('assignedTrustedAppsListStateChanged', { + validate({ payload }) { + return isLoadedResourceState(payload); + }, + }), + ]) + : Promise.resolve(); + + appTestContext.history.push( + getPolicyDetailsArtifactsListPath('ddf6570b-9175-4a6d-b288-61a09771c647') + ); + renderResult = appTestContext.render( + + ); + + await pendingDataLoadState; + + return renderResult; + }; + }); + + const getConfirmButton = (): HTMLButtonElement => + renderResult.getByTestId('confirmModalConfirmButton') as HTMLButtonElement; + + const clickConfirmButton = async ( + waitForActionDispatch: boolean = false + ): Promise | undefined> => { + const pendingConfirmStoreAction = waitForAction('policyArtifactsUpdateTrustedApps'); + + act(() => { + fireEvent.click(getConfirmButton()); + }); + + let response: PolicyArtifactsUpdateTrustedApps | undefined; + + if (waitForActionDispatch) { + await act(async () => { + response = await pendingConfirmStoreAction; + }); + } + + return response; + }; + + const clickCancelButton = () => { + act(() => { + fireEvent.click(renderResult.getByTestId('confirmModalCancelButton')); + }); + }; + + const clickCloseButton = () => { + act(() => { + fireEvent.click(renderResult.baseElement.querySelector('button.euiModal__closeIcon')!); + }); + }; + + it.each([ + ['cancel', clickCancelButton], + ['close', clickCloseButton], + ])('should call `onClose` callback when %s button is clicked', async (__, clickButton) => { + await render(); + clickButton(); + + expect(onCloseHandler).toHaveBeenCalled(); + }); + + it('should dispatch action when confirmed', async () => { + await render(); + const confirmedAction = await clickConfirmButton(true); + + expect(confirmedAction!.payload).toEqual({ + action: 'remove', + artifacts: trustedApps, + }); + }); + + it.skip('should disable and show loading state on confirm button while update is underway', async () => { + await render(); + await clickConfirmButton(true); + const confirmButton = getConfirmButton(); + + // FIXME:PT will finish test in a subsequent PR (issue created #1876) + // GETTING ERROR: + // Error: current policy id not found + // // at removeTrustedAppsFromPolicy (.../x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware/policy_trusted_apps_middleware.ts:368:13) + // // at policyTrustedAppsMiddlewareRunner (.../x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware/policy_trusted_apps_middleware.ts:93:9) + + expect(confirmButton.disabled).toBe(true); + expect(confirmButton.querySelector('.euiLoadingSpinner')).not.toBeNull(); + }); + + it.each([ + ['cancel', clickCancelButton], + ['close', clickCloseButton], + ])( + 'should prevent dialog dismissal if %s button is clicked while update is underway', + (__, clickButton) => { + // TODO: implement test + } + ); + + it.todo('should show error toast if removal failed'); + + it.todo('should show success toast and close modal when removed is successful'); + + it.todo('should show single removal success message'); + + it.todo('should show multiples removal success message'); +}); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/remove_trusted_app_from_policy_modal.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/remove_trusted_app_from_policy_modal.tsx new file mode 100644 index 0000000000000..c43c28ec82829 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/trusted_apps/list/remove_trusted_app_from_policy_modal.tsx @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo, useCallback, useEffect, useMemo } from 'react'; +import { EuiCallOut, EuiConfirmModal, EuiSpacer, EuiText } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { useDispatch } from 'react-redux'; +import { Dispatch } from 'redux'; +import { Immutable, TrustedApp } from '../../../../../../../common/endpoint/types'; +import { AppAction } from '../../../../../../common/store/actions'; +import { usePolicyDetailsSelector } from '../../policy_hooks'; +import { + getTrustedAppsIsRemoving, + getTrustedAppsRemovalError, + getTrustedAppsWasRemoveSuccessful, + policyDetails, +} from '../../../store/policy_details/selectors'; +import { useToasts } from '../../../../../../common/lib/kibana'; + +export interface RemoveTrustedAppFromPolicyModalProps { + trustedApps: Immutable; + onClose: () => void; +} + +export const RemoveTrustedAppFromPolicyModal = memo( + ({ trustedApps, onClose }) => { + const toasts = useToasts(); + const dispatch = useDispatch>(); + + const policyName = usePolicyDetailsSelector(policyDetails)?.name; + const isRemoving = usePolicyDetailsSelector(getTrustedAppsIsRemoving); + const removeError = usePolicyDetailsSelector(getTrustedAppsRemovalError); + const wasSuccessful = usePolicyDetailsSelector(getTrustedAppsWasRemoveSuccessful); + + const removedToastMessage: string = useMemo(() => { + const count = trustedApps.length; + + if (count === 0) { + return ''; + } + + if (count > 1) { + return i18n.translate( + 'xpack.securitySolution.endpoint.policy.trustedApps.list.removeDialog.successMultiplesToastText', + { + defaultMessage: '{count} trusted apps have been removed from {policyName} policy', + values: { count, policyName }, + } + ); + } + + return i18n.translate( + 'xpack.securitySolution.endpoint.policy.trustedApps.list.removeDialog.successToastText', + { + defaultMessage: '"{trustedAppName}" has been removed from "{policyName}" policy', + values: { trustedAppName: trustedApps[0].name, policyName }, + } + ); + }, [policyName, trustedApps]); + + const handleModalClose = useCallback(() => { + if (!isRemoving) { + onClose(); + } + }, [isRemoving, onClose]); + + const handleModalConfirm = useCallback(() => { + dispatch({ + type: 'policyArtifactsUpdateTrustedApps', + payload: { action: 'remove', artifacts: trustedApps }, + }); + }, [dispatch, trustedApps]); + + useEffect(() => { + // When component is un-mounted, reset the state for remove in the store + return () => { + dispatch({ type: 'policyDetailsArtifactsResetRemove' }); + }; + }, [dispatch]); + + useEffect(() => { + if (removeError) { + toasts.addError(removeError as unknown as Error, { + title: i18n.translate( + 'xpack.securitySolution.endpoint.policy.trustedApps.list.removeDialog.errorToastTitle', + { + defaultMessage: 'Error while attempt to remove trusted application', + } + ), + }); + } + }, [removeError, toasts]); + + useEffect(() => { + if (wasSuccessful) { + toasts.addSuccess({ + title: i18n.translate( + 'xpack.securitySolution.endpoint.policy.trustedApps.list.removeDialog.successToastTitle', + { defaultMessage: 'Successfully removed' } + ), + text: removedToastMessage, + }); + handleModalClose(); + } + }, [handleModalClose, policyName, removedToastMessage, toasts, trustedApps, wasSuccessful]); + + return ( + + +

+ +

+ + + + + +

+ +

+ + + ); + } +); +RemoveTrustedAppFromPolicyModal.displayName = 'RemoveTrustedAppFromPolicyModal'; diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/service/index.ts b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/service/index.ts index 09aa80ffae495..b59fb6cfdd2f7 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/service/index.ts +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/service/index.ts @@ -7,6 +7,7 @@ import { HttpStart } from 'kibana/public'; +import pMap from 'p-map'; import { TRUSTED_APPS_CREATE_API, TRUSTED_APPS_DELETE_API, @@ -29,10 +30,14 @@ import { GetOneTrustedAppRequestParams, GetOneTrustedAppResponse, GetTrustedAppsSummaryRequest, -} from '../../../../../common/endpoint/types/trusted_apps'; + TrustedApp, + MaybeImmutable, +} from '../../../../../common/endpoint/types'; import { resolvePathVariables } from '../../../../common/utils/resolve_path_variables'; import { sendGetEndpointSpecificPackagePolicies } from '../../policy/store/services/ingest'; +import { toUpdateTrustedApp } from '../../../../../common/endpoint/service/trusted_apps/to_update_trusted_app'; +import { isGlobalEffectScope } from '../state/type_guards'; export interface TrustedAppsService { getTrustedApp(params: GetOneTrustedAppRequestParams): Promise; @@ -46,8 +51,23 @@ export interface TrustedAppsService { getPolicyList( options?: Parameters[1] ): ReturnType; + assignPolicyToTrustedApps( + policyId: string, + trustedApps: MaybeImmutable + ): Promise; + removePolicyFromTrustedApps( + policyId: string, + trustedApps: MaybeImmutable + ): Promise; } +const P_MAP_OPTIONS = Object.freeze({ + concurrency: 5, + /** When set to false, instead of stopping when a promise rejects, it will wait for all the promises to settle + * and then reject with an aggregated error containing all the errors from the rejected promises. */ + stopOnError: false, +}); + export class TrustedAppsHttpService implements TrustedAppsService { constructor(private http: HttpStart) {} @@ -92,4 +112,79 @@ export class TrustedAppsHttpService implements TrustedAppsService { getPolicyList(options?: Parameters[1]) { return sendGetEndpointSpecificPackagePolicies(this.http, options); } + + /** + * Assign a policy to trusted apps. Note that Trusted Apps MUST NOT be global + * + * @param policyId + * @param trustedApps[] + */ + assignPolicyToTrustedApps( + policyId: string, + trustedApps: MaybeImmutable + ): Promise { + return this._handleAssignOrRemovePolicyId('assign', policyId, trustedApps); + } + + /** + * Remove a policy from trusted apps. Note that Trusted Apps MUST NOT be global + * + * @param policyId + * @param trustedApps[] + */ + removePolicyFromTrustedApps( + policyId: string, + trustedApps: MaybeImmutable + ): Promise { + return this._handleAssignOrRemovePolicyId('remove', policyId, trustedApps); + } + + private _handleAssignOrRemovePolicyId( + action: 'assign' | 'remove', + policyId: string, + trustedApps: MaybeImmutable + ): Promise { + if (policyId.trim() === '') { + throw new Error('policy ID is required'); + } + + if (trustedApps.length === 0) { + throw new Error('at least one trusted app is required'); + } + + return pMap( + trustedApps, + async (trustedApp) => { + if (isGlobalEffectScope(trustedApp.effectScope)) { + throw new Error( + `Unable to update trusted app [${trustedApp.id}] policy assignment. It's effectScope is 'global'` + ); + } + + const policies: string[] = !isGlobalEffectScope(trustedApp.effectScope) + ? [...trustedApp.effectScope.policies] + : []; + + const indexOfPolicyId = policies.indexOf(policyId); + + if (action === 'assign' && indexOfPolicyId === -1) { + policies.push(policyId); + } else if (action === 'remove' && indexOfPolicyId !== -1) { + policies.splice(indexOfPolicyId, 1); + } + + return this.updateTrustedApp( + { id: trustedApp.id }, + { + ...toUpdateTrustedApp(trustedApp), + effectScope: { + type: 'policy', + policies, + }, + } + ); + }, + P_MAP_OPTIONS + ); + } } diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/middleware.test.ts b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/middleware.test.ts index f64003ec6ad91..4455baddb047c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/middleware.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/middleware.test.ts @@ -54,6 +54,8 @@ const createTrustedAppsServiceMock = (): jest.Mocked => ({ getPolicyList: jest.fn(), updateTrustedApp: jest.fn(), getTrustedApp: jest.fn(), + assignPolicyToTrustedApps: jest.fn(), + removePolicyFromTrustedApps: jest.fn(), }); const createStoreSetup = (trustedAppsService: TrustedAppsService) => { From eeed2ca6ae6af0b2ffcde9beccf166e1462a31c9 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 13 Oct 2021 11:03:04 -0500 Subject: [PATCH 04/35] [ci] filter out proc-runner logs from stdout on CI (#114568) Co-authored-by: spalger --- .../src/ci_stats_reporter/ci_stats_config.ts | 10 +++ .../src/ci_stats_reporter/index.ts | 1 + .../src/proc_runner/proc_runner.ts | 2 + packages/kbn-dev-utils/src/run/cleanup.ts | 4 + packages/kbn-dev-utils/src/run/index.ts | 1 + .../__snapshots__/tooling_log.test.ts.snap | 19 +++++ .../kbn-dev-utils/src/tooling_log/index.ts | 3 + .../kbn-dev-utils/src/tooling_log/message.ts | 8 ++ .../src/tooling_log/tooling_log.test.ts | 37 +++++++++ .../src/tooling_log/tooling_log.ts | 66 ++++++++++++---- .../tooling_log_collecting_writer.ts | 15 ++++ .../tooling_log/tooling_log_text_writer.ts | 19 +++++ .../kbn-dev-utils/src/tooling_log/writer.ts | 9 +++ packages/kbn-es/src/cluster.js | 2 +- .../src/integration_tests/cluster.test.js | 68 +++++++++++++---- packages/kbn-pm/dist/index.js | 75 +++++++++++++++---- .../lib/mocha/reporter/reporter.js | 3 +- .../kbn-test/src/functional_tests/tasks.ts | 7 +- test/functional/services/remote/remote.ts | 2 +- 19 files changed, 303 insertions(+), 48 deletions(-) diff --git a/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_config.ts b/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_config.ts index 9af52ae8d2df0..f73b9c830a2ab 100644 --- a/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_config.ts +++ b/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_config.ts @@ -8,8 +8,18 @@ import type { ToolingLog } from '../tooling_log'; +/** + * Information about how CiStatsReporter should talk to the ci-stats service. Normally + * it is read from a JSON environment variable using the `parseConfig()` function + * exported by this module. + */ export interface Config { + /** ApiToken necessary for writing build data to ci-stats service */ apiToken: string; + /** + * uuid which should be obtained by first creating a build with the + * ci-stats service and then passing it to all subsequent steps + */ buildId: string; } diff --git a/packages/kbn-dev-utils/src/ci_stats_reporter/index.ts b/packages/kbn-dev-utils/src/ci_stats_reporter/index.ts index 9cb05608526eb..318a2921517f1 100644 --- a/packages/kbn-dev-utils/src/ci_stats_reporter/index.ts +++ b/packages/kbn-dev-utils/src/ci_stats_reporter/index.ts @@ -7,5 +7,6 @@ */ export * from './ci_stats_reporter'; +export type { Config } from './ci_stats_config'; export * from './ship_ci_stats_cli'; export { getTimeReporter } from './report_time'; diff --git a/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts b/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts index 35c910c911105..8ef32411621f8 100644 --- a/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts +++ b/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts @@ -37,6 +37,8 @@ export class ProcRunner { private signalUnsubscribe: () => void; constructor(private log: ToolingLog) { + this.log = log.withType('ProcRunner'); + this.signalUnsubscribe = exitHook(() => { this.teardown().catch((error) => { log.error(`ProcRunner teardown error: ${error.stack}`); diff --git a/packages/kbn-dev-utils/src/run/cleanup.ts b/packages/kbn-dev-utils/src/run/cleanup.ts index 26a6f5c632c46..ba0b762a562ad 100644 --- a/packages/kbn-dev-utils/src/run/cleanup.ts +++ b/packages/kbn-dev-utils/src/run/cleanup.ts @@ -13,6 +13,10 @@ import exitHook from 'exit-hook'; import { ToolingLog } from '../tooling_log'; import { isFailError } from './fail'; +/** + * A function which will be called when the CLI is torn-down which should + * quickly cleanup whatever it needs. + */ export type CleanupTask = () => void; export class Cleanup { diff --git a/packages/kbn-dev-utils/src/run/index.ts b/packages/kbn-dev-utils/src/run/index.ts index f3c364c774d30..505ef4ee264d6 100644 --- a/packages/kbn-dev-utils/src/run/index.ts +++ b/packages/kbn-dev-utils/src/run/index.ts @@ -10,3 +10,4 @@ export * from './run'; export * from './run_with_commands'; export * from './flags'; export * from './fail'; +export type { CleanupTask } from './cleanup'; diff --git a/packages/kbn-dev-utils/src/tooling_log/__snapshots__/tooling_log.test.ts.snap b/packages/kbn-dev-utils/src/tooling_log/__snapshots__/tooling_log.test.ts.snap index 059e3d49c3688..7742c2bb681d0 100644 --- a/packages/kbn-dev-utils/src/tooling_log/__snapshots__/tooling_log.test.ts.snap +++ b/packages/kbn-dev-utils/src/tooling_log/__snapshots__/tooling_log.test.ts.snap @@ -10,6 +10,7 @@ Array [ "baz", ], "indent": 0, + "source": undefined, "type": "debug", }, ], @@ -24,6 +25,7 @@ Array [ [Error: error message], ], "indent": 0, + "source": undefined, "type": "error", }, ], @@ -33,6 +35,7 @@ Array [ "string message", ], "indent": 0, + "source": undefined, "type": "error", }, ], @@ -50,6 +53,7 @@ Array [ "foo", ], "indent": 0, + "source": undefined, "type": "debug", }, Object { @@ -57,6 +61,7 @@ Array [ "bar", ], "indent": 0, + "source": undefined, "type": "info", }, Object { @@ -64,6 +69,7 @@ Array [ "baz", ], "indent": 0, + "source": undefined, "type": "verbose", }, ] @@ -76,6 +82,7 @@ Array [ "foo", ], "indent": 0, + "source": undefined, "type": "debug", }, Object { @@ -83,6 +90,7 @@ Array [ "bar", ], "indent": 0, + "source": undefined, "type": "info", }, Object { @@ -90,6 +98,7 @@ Array [ "baz", ], "indent": 0, + "source": undefined, "type": "verbose", }, ] @@ -103,6 +112,7 @@ Array [ "foo", ], "indent": 1, + "source": undefined, "type": "debug", }, ], @@ -112,6 +122,7 @@ Array [ "bar", ], "indent": 3, + "source": undefined, "type": "debug", }, ], @@ -121,6 +132,7 @@ Array [ "baz", ], "indent": 6, + "source": undefined, "type": "debug", }, ], @@ -130,6 +142,7 @@ Array [ "box", ], "indent": 4, + "source": undefined, "type": "debug", }, ], @@ -139,6 +152,7 @@ Array [ "foo", ], "indent": 0, + "source": undefined, "type": "debug", }, ], @@ -155,6 +169,7 @@ Array [ "baz", ], "indent": 0, + "source": undefined, "type": "info", }, ], @@ -171,6 +186,7 @@ Array [ "baz", ], "indent": 0, + "source": undefined, "type": "success", }, ], @@ -187,6 +203,7 @@ Array [ "baz", ], "indent": 0, + "source": undefined, "type": "verbose", }, ], @@ -203,6 +220,7 @@ Array [ "baz", ], "indent": 0, + "source": undefined, "type": "warning", }, ], @@ -219,6 +237,7 @@ Array [ "baz", ], "indent": 0, + "source": undefined, "type": "write", }, ], diff --git a/packages/kbn-dev-utils/src/tooling_log/index.ts b/packages/kbn-dev-utils/src/tooling_log/index.ts index 65dcd3054ef93..4da54ee9bfeae 100644 --- a/packages/kbn-dev-utils/src/tooling_log/index.ts +++ b/packages/kbn-dev-utils/src/tooling_log/index.ts @@ -7,6 +7,9 @@ */ export { ToolingLog } from './tooling_log'; +export type { ToolingLogOptions } from './tooling_log'; export { ToolingLogTextWriter, ToolingLogTextWriterConfig } from './tooling_log_text_writer'; export { pickLevelFromFlags, parseLogLevel, LogLevel, ParsedLogLevel } from './log_levels'; export { ToolingLogCollectingWriter } from './tooling_log_collecting_writer'; +export type { Writer } from './writer'; +export type { Message } from './message'; diff --git a/packages/kbn-dev-utils/src/tooling_log/message.ts b/packages/kbn-dev-utils/src/tooling_log/message.ts index ebd3a255a73a4..082c0e65d48b2 100644 --- a/packages/kbn-dev-utils/src/tooling_log/message.ts +++ b/packages/kbn-dev-utils/src/tooling_log/message.ts @@ -8,8 +8,16 @@ export type MessageTypes = 'verbose' | 'debug' | 'info' | 'success' | 'warning' | 'error' | 'write'; +/** + * The object shape passed to ToolingLog writers each time the log is used. + */ export interface Message { + /** level/type of message */ type: MessageTypes; + /** indentation intended when message written to a text log */ indent: number; + /** type of logger this message came from */ + source?: string; + /** args passed to the logging method */ args: any[]; } diff --git a/packages/kbn-dev-utils/src/tooling_log/tooling_log.test.ts b/packages/kbn-dev-utils/src/tooling_log/tooling_log.test.ts index ec63a9fb7e6f2..506f89786917f 100644 --- a/packages/kbn-dev-utils/src/tooling_log/tooling_log.test.ts +++ b/packages/kbn-dev-utils/src/tooling_log/tooling_log.test.ts @@ -155,3 +155,40 @@ describe('#getWritten$()', () => { await testWrittenMsgs([{ write: jest.fn(() => false) }, { write: jest.fn(() => false) }]); }); }); + +describe('#withType()', () => { + it('creates a child logger with a unique type that respects all other settings', () => { + const writerA = new ToolingLogCollectingWriter(); + const writerB = new ToolingLogCollectingWriter(); + const log = new ToolingLog(); + log.setWriters([writerA]); + + const fork = log.withType('someType'); + log.info('hello'); + fork.info('world'); + fork.indent(2); + log.debug('indented'); + fork.indent(-2); + log.debug('not-indented'); + + log.setWriters([writerB]); + fork.info('to new writer'); + fork.indent(5); + log.info('also to new writer'); + + expect(writerA.messages).toMatchInlineSnapshot(` + Array [ + " info hello", + " info source[someType] world", + " │ debg indented", + " debg not-indented", + ] + `); + expect(writerB.messages).toMatchInlineSnapshot(` + Array [ + " info source[someType] to new writer", + " │ info also to new writer", + ] + `); + }); +}); diff --git a/packages/kbn-dev-utils/src/tooling_log/tooling_log.ts b/packages/kbn-dev-utils/src/tooling_log/tooling_log.ts index e9fd15afefe4e..84e9159dfcd41 100644 --- a/packages/kbn-dev-utils/src/tooling_log/tooling_log.ts +++ b/packages/kbn-dev-utils/src/tooling_log/tooling_log.ts @@ -12,21 +12,45 @@ import { ToolingLogTextWriter, ToolingLogTextWriterConfig } from './tooling_log_ import { Writer } from './writer'; import { Message, MessageTypes } from './message'; +export interface ToolingLogOptions { + /** + * type name for this logger, will be assigned to the "source" + * properties of messages produced by this logger + */ + type?: string; + /** + * parent ToolingLog. When a ToolingLog has a parent they will both + * share indent and writers state. Changing the indent width or + * writers on either log will update the other too. + */ + parent?: ToolingLog; +} + export class ToolingLog { - private indentWidth = 0; - private writers: Writer[]; + private indentWidth$: Rx.BehaviorSubject; + private writers$: Rx.BehaviorSubject; private readonly written$: Rx.Subject; + private readonly type: string | undefined; + + constructor(writerConfig?: ToolingLogTextWriterConfig, options?: ToolingLogOptions) { + this.indentWidth$ = options?.parent ? options.parent.indentWidth$ : new Rx.BehaviorSubject(0); - constructor(writerConfig?: ToolingLogTextWriterConfig) { - this.writers = writerConfig ? [new ToolingLogTextWriter(writerConfig)] : []; - this.written$ = new Rx.Subject(); + this.writers$ = options?.parent + ? options.parent.writers$ + : new Rx.BehaviorSubject([]); + if (!options?.parent && writerConfig) { + this.writers$.next([new ToolingLogTextWriter(writerConfig)]); + } + + this.written$ = options?.parent ? options.parent.written$ : new Rx.Subject(); + this.type = options?.type; } /** * Get the current indentation level of the ToolingLog */ public getIndent() { - return this.indentWidth; + return this.indentWidth$.getValue(); } /** @@ -39,8 +63,8 @@ export class ToolingLog { * @param block a function to run and reset any indentation changes after */ public indent(delta = 0, block?: () => Promise) { - const originalWidth = this.indentWidth; - this.indentWidth = Math.max(this.indentWidth + delta, 0); + const originalWidth = this.indentWidth$.getValue(); + this.indentWidth$.next(Math.max(originalWidth + delta, 0)); if (!block) { return; } @@ -49,7 +73,7 @@ export class ToolingLog { try { return await block(); } finally { - this.indentWidth = originalWidth; + this.indentWidth$.next(originalWidth); } })(); } @@ -83,26 +107,40 @@ export class ToolingLog { } public getWriters() { - return this.writers.slice(0); + return [...this.writers$.getValue()]; } public setWriters(writers: Writer[]) { - this.writers = [...writers]; + this.writers$.next([...writers]); } public getWritten$() { return this.written$.asObservable(); } + /** + * Create a new ToolingLog which sets a different "type", allowing messages to be filtered out by "source" + * @param type A string that will be passed along with messages from this logger which can be used to filter messages with `ignoreSources` + */ + public withType(type: string) { + return new ToolingLog(undefined, { + type, + parent: this, + }); + } + private sendToWriters(type: MessageTypes, args: any[]) { - const msg = { + const indent = this.indentWidth$.getValue(); + const writers = this.writers$.getValue(); + const msg: Message = { type, - indent: this.indentWidth, + indent, + source: this.type, args, }; let written = false; - for (const writer of this.writers) { + for (const writer of writers) { if (writer.write(msg)) { written = true; } diff --git a/packages/kbn-dev-utils/src/tooling_log/tooling_log_collecting_writer.ts b/packages/kbn-dev-utils/src/tooling_log/tooling_log_collecting_writer.ts index cc399e40d2cb4..6f73563f4a2c5 100644 --- a/packages/kbn-dev-utils/src/tooling_log/tooling_log_collecting_writer.ts +++ b/packages/kbn-dev-utils/src/tooling_log/tooling_log_collecting_writer.ts @@ -8,6 +8,7 @@ import { ToolingLogTextWriter } from './tooling_log_text_writer'; import { LogLevel } from './log_levels'; +import { Message } from './message'; export class ToolingLogCollectingWriter extends ToolingLogTextWriter { messages: string[] = []; @@ -23,4 +24,18 @@ export class ToolingLogCollectingWriter extends ToolingLogTextWriter { }, }); } + + /** + * Called by ToolingLog, extends messages with the source if message includes one. + */ + write(msg: Message) { + if (msg.source) { + return super.write({ + ...msg, + args: [`source[${msg.source}]`, ...msg.args], + }); + } + + return super.write(msg); + } } diff --git a/packages/kbn-dev-utils/src/tooling_log/tooling_log_text_writer.ts b/packages/kbn-dev-utils/src/tooling_log/tooling_log_text_writer.ts index 2b1806eb4b9a2..660dae3fa1f55 100644 --- a/packages/kbn-dev-utils/src/tooling_log/tooling_log_text_writer.ts +++ b/packages/kbn-dev-utils/src/tooling_log/tooling_log_text_writer.ts @@ -28,7 +28,20 @@ const MSG_PREFIXES = { const has = (obj: T, key: any): key is keyof T => obj.hasOwnProperty(key); export interface ToolingLogTextWriterConfig { + /** + * Log level, messages below this level will be ignored + */ level: LogLevel; + /** + * List of message sources/ToolingLog types which will be ignored. Create + * a logger with `ToolingLog#withType()` to create messages with a specific + * source. Ignored messages will be dropped without writing. + */ + ignoreSources?: string[]; + /** + * Target which will receive formatted message lines, a common value for `writeTo` + * is process.stdout + */ writeTo: { write(s: string): void; }; @@ -59,10 +72,12 @@ export class ToolingLogTextWriter implements Writer { public readonly writeTo: { write(msg: string): void; }; + private readonly ignoreSources?: string[]; constructor(config: ToolingLogTextWriterConfig) { this.level = parseLogLevel(config.level); this.writeTo = config.writeTo; + this.ignoreSources = config.ignoreSources; if (!this.writeTo || typeof this.writeTo.write !== 'function') { throw new Error( @@ -76,6 +91,10 @@ export class ToolingLogTextWriter implements Writer { return false; } + if (this.ignoreSources && msg.source && this.ignoreSources.includes(msg.source)) { + return false; + } + const prefix = has(MSG_PREFIXES, msg.type) ? MSG_PREFIXES[msg.type] : ''; ToolingLogTextWriter.write(this.writeTo, prefix, msg); return true; diff --git a/packages/kbn-dev-utils/src/tooling_log/writer.ts b/packages/kbn-dev-utils/src/tooling_log/writer.ts index fd56f4fe3d3a6..26fec6a780694 100644 --- a/packages/kbn-dev-utils/src/tooling_log/writer.ts +++ b/packages/kbn-dev-utils/src/tooling_log/writer.ts @@ -8,6 +8,15 @@ import { Message } from './message'; +/** + * An object which received ToolingLog `Messages` and sends them to + * some interface for collecting logs like stdio, or a file + */ export interface Writer { + /** + * Called with every log message, should return true if the message + * was written and false if it was ignored. + * @param msg The log message to write + */ write(msg: Message): boolean; } diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 0866b14f4ade8..dd9c17055fb18 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -36,7 +36,7 @@ const first = (stream, map) => exports.Cluster = class Cluster { constructor({ log = defaultLog, ssl = false } = {}) { - this._log = log; + this._log = log.withType('@kbn/es Cluster'); this._ssl = ssl; this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined; } diff --git a/packages/kbn-es/src/integration_tests/cluster.test.js b/packages/kbn-es/src/integration_tests/cluster.test.js index c196a89a6b090..0cdbac310bbb1 100644 --- a/packages/kbn-es/src/integration_tests/cluster.test.js +++ b/packages/kbn-es/src/integration_tests/cluster.test.js @@ -8,9 +8,11 @@ const { ToolingLog, + ToolingLogCollectingWriter, ES_P12_PATH, ES_P12_PASSWORD, createAnyInstanceSerializer, + createStripAnsiSerializer, } = require('@kbn/dev-utils'); const execa = require('execa'); const { Cluster } = require('../cluster'); @@ -18,6 +20,7 @@ const { installSource, installSnapshot, installArchive } = require('../install') const { extractConfigFiles } = require('../utils/extract_config_files'); expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog)); +expect.addSnapshotSerializer(createStripAnsiSerializer()); jest.mock('../install', () => ({ installSource: jest.fn(), @@ -31,6 +34,8 @@ jest.mock('../utils/extract_config_files', () => ({ })); const log = new ToolingLog(); +const logWriter = new ToolingLogCollectingWriter(); +log.setWriters([logWriter]); function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -76,6 +81,8 @@ const initialEnv = { ...process.env }; beforeEach(() => { jest.resetAllMocks(); extractConfigFiles.mockImplementation((config) => config); + log.indent(-log.getIndent()); + logWriter.messages.length = 0; }); afterEach(() => { @@ -107,11 +114,21 @@ describe('#installSource()', () => { installSource.mockResolvedValue({}); const cluster = new Cluster({ log }); await cluster.installSource({ foo: 'bar' }); - expect(installSource).toHaveBeenCalledTimes(1); - expect(installSource).toHaveBeenCalledWith({ - log, - foo: 'bar', - }); + expect(installSource.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "foo": "bar", + "log": , + }, + ], + ] + `); + expect(logWriter.messages).toMatchInlineSnapshot(` + Array [ + " info source[@kbn/es Cluster] Installing from source", + ] + `); }); it('rejects if installSource() rejects', async () => { @@ -146,11 +163,21 @@ describe('#installSnapshot()', () => { installSnapshot.mockResolvedValue({}); const cluster = new Cluster({ log }); await cluster.installSnapshot({ foo: 'bar' }); - expect(installSnapshot).toHaveBeenCalledTimes(1); - expect(installSnapshot).toHaveBeenCalledWith({ - log, - foo: 'bar', - }); + expect(installSnapshot.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "foo": "bar", + "log": , + }, + ], + ] + `); + expect(logWriter.messages).toMatchInlineSnapshot(` + Array [ + " info source[@kbn/es Cluster] Installing from snapshot", + ] + `); }); it('rejects if installSnapshot() rejects', async () => { @@ -185,11 +212,22 @@ describe('#installArchive(path)', () => { installArchive.mockResolvedValue({}); const cluster = new Cluster({ log }); await cluster.installArchive('path', { foo: 'bar' }); - expect(installArchive).toHaveBeenCalledTimes(1); - expect(installArchive).toHaveBeenCalledWith('path', { - log, - foo: 'bar', - }); + expect(installArchive.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "path", + Object { + "foo": "bar", + "log": , + }, + ], + ] + `); + expect(logWriter.messages).toMatchInlineSnapshot(` + Array [ + " info source[@kbn/es Cluster] Installing from an archive", + ] + `); }); it('rejects if installArchive() rejects', async () => { diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index 595b619a7f2a4..721072d9e899b 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -625,12 +625,20 @@ function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && * Side Public License, v 1. */ class ToolingLog { - constructor(writerConfig) { - (0, _defineProperty2.default)(this, "indentWidth", 0); - (0, _defineProperty2.default)(this, "writers", void 0); + constructor(writerConfig, options) { + (0, _defineProperty2.default)(this, "indentWidth$", void 0); + (0, _defineProperty2.default)(this, "writers$", void 0); (0, _defineProperty2.default)(this, "written$", void 0); - this.writers = writerConfig ? [new _tooling_log_text_writer.ToolingLogTextWriter(writerConfig)] : []; - this.written$ = new Rx.Subject(); + (0, _defineProperty2.default)(this, "type", void 0); + this.indentWidth$ = options !== null && options !== void 0 && options.parent ? options.parent.indentWidth$ : new Rx.BehaviorSubject(0); + this.writers$ = options !== null && options !== void 0 && options.parent ? options.parent.writers$ : new Rx.BehaviorSubject([]); + + if (!(options !== null && options !== void 0 && options.parent) && writerConfig) { + this.writers$.next([new _tooling_log_text_writer.ToolingLogTextWriter(writerConfig)]); + } + + this.written$ = options !== null && options !== void 0 && options.parent ? options.parent.written$ : new Rx.Subject(); + this.type = options === null || options === void 0 ? void 0 : options.type; } /** * Get the current indentation level of the ToolingLog @@ -638,7 +646,7 @@ class ToolingLog { getIndent() { - return this.indentWidth; + return this.indentWidth$.getValue(); } /** * Indent the output of the ToolingLog by some character (4 is a good choice usually). @@ -652,8 +660,8 @@ class ToolingLog { indent(delta = 0, block) { - const originalWidth = this.indentWidth; - this.indentWidth = Math.max(this.indentWidth + delta, 0); + const originalWidth = this.indentWidth$.getValue(); + this.indentWidth$.next(Math.max(originalWidth + delta, 0)); if (!block) { return; @@ -663,7 +671,7 @@ class ToolingLog { try { return await block(); } finally { - this.indentWidth = originalWidth; + this.indentWidth$.next(originalWidth); } })(); } @@ -697,26 +705,41 @@ class ToolingLog { } getWriters() { - return this.writers.slice(0); + return [...this.writers$.getValue()]; } setWriters(writers) { - this.writers = [...writers]; + this.writers$.next([...writers]); } getWritten$() { return this.written$.asObservable(); } + /** + * Create a new ToolingLog which sets a different "type", allowing messages to be filtered out by "source" + * @param type A string that will be passed along with messages from this logger which can be used to filter messages with `ignoreSources` + */ + + + withType(type) { + return new ToolingLog(undefined, { + type, + parent: this + }); + } sendToWriters(type, args) { + const indent = this.indentWidth$.getValue(); + const writers = this.writers$.getValue(); const msg = { type, - indent: this.indentWidth, + indent, + source: this.type, args }; let written = false; - for (const writer of this.writers) { + for (const writer of writers) { if (writer.write(msg)) { written = true; } @@ -6618,8 +6641,10 @@ class ToolingLogTextWriter { constructor(config) { (0, _defineProperty2.default)(this, "level", void 0); (0, _defineProperty2.default)(this, "writeTo", void 0); + (0, _defineProperty2.default)(this, "ignoreSources", void 0); this.level = (0, _log_levels.parseLogLevel)(config.level); this.writeTo = config.writeTo; + this.ignoreSources = config.ignoreSources; if (!this.writeTo || typeof this.writeTo.write !== 'function') { throw new Error('ToolingLogTextWriter requires the `writeTo` option be set to a stream (like process.stdout)'); @@ -6631,6 +6656,10 @@ class ToolingLogTextWriter { return false; } + if (this.ignoreSources && msg.source && this.ignoreSources.includes(msg.source)) { + return false; + } + const prefix = has(MSG_PREFIXES, msg.type) ? MSG_PREFIXES[msg.type] : ''; ToolingLogTextWriter.write(this.writeTo, prefix, msg); return true; @@ -8773,6 +8802,20 @@ class ToolingLogCollectingWriter extends _tooling_log_text_writer.ToolingLogText }); (0, _defineProperty2.default)(this, "messages", []); } + /** + * Called by ToolingLog, extends messages with the source if message includes one. + */ + + + write(msg) { + if (msg.source) { + return super.write({ ...msg, + args: [`source[${msg.source}]`, ...msg.args] + }); + } + + return super.write(msg); + } } @@ -15466,6 +15509,12 @@ exports.parseConfig = parseConfig; * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ + +/** + * Information about how CiStatsReporter should talk to the ci-stats service. Normally + * it is read from a JSON environment variable using the `parseConfig()` function + * exported by this module. + */ function validateConfig(log, config) { const validApiToken = typeof config.apiToken === 'string' && config.apiToken.length !== 0; diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js index 43229ff2d1c98..d6045b71bf3a7 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js @@ -62,6 +62,7 @@ export function MochaReporterProvider({ getService }) { log.setWriters([ new ToolingLogTextWriter({ level: 'error', + ignoreSources: ['ProcRunner', '@kbn/es Cluster'], writeTo: process.stdout, }), new ToolingLogTextWriter({ @@ -136,7 +137,7 @@ export function MochaReporterProvider({ getService }) { onPass = (test) => { const time = colors.speed(test.speed, ` (${ms(test.duration)})`); const pass = colors.pass(`${symbols.ok} pass`); - log.write(`- ${pass} ${time} "${test.fullTitle()}"`); + log.write(`- ${pass} ${time}`); }; onFail = (runnable) => { diff --git a/packages/kbn-test/src/functional_tests/tasks.ts b/packages/kbn-test/src/functional_tests/tasks.ts index c8265c032cbcc..b220c3899a638 100644 --- a/packages/kbn-test/src/functional_tests/tasks.ts +++ b/packages/kbn-test/src/functional_tests/tasks.ts @@ -90,7 +90,7 @@ export async function runTests(options: RunTestsParams) { log.write('--- determining which ftr configs to run'); const configPathsWithTests: string[] = []; for (const configPath of options.configs) { - log.info('testing', configPath); + log.info('testing', relative(REPO_ROOT, configPath)); await log.indent(4, async () => { if (await hasTests({ configPath, options: { ...options, log } })) { configPathsWithTests.push(configPath); @@ -98,9 +98,10 @@ export async function runTests(options: RunTestsParams) { }); } - for (const configPath of configPathsWithTests) { + for (const [i, configPath] of configPathsWithTests.entries()) { await log.indent(0, async () => { - log.write(`--- Running ${relative(REPO_ROOT, configPath)}`); + const progress = `${i + 1}/${configPathsWithTests.length}`; + log.write(`--- [${progress}] Running ${relative(REPO_ROOT, configPath)}`); await withProcRunner(log, async (procs) => { const config = await readConfigFile(log, configPath); diff --git a/test/functional/services/remote/remote.ts b/test/functional/services/remote/remote.ts index 5bf99b4bf1136..653058959b839 100644 --- a/test/functional/services/remote/remote.ts +++ b/test/functional/services/remote/remote.ts @@ -92,7 +92,7 @@ export async function RemoteProvider({ getService }: FtrProviderContext) { .subscribe({ next({ message, level }) { const msg = message.replace(/\\n/g, '\n'); - log[level === 'SEVERE' || level === 'error' ? 'error' : 'debug']( + log[level === 'SEVERE' || level === 'error' ? 'warning' : 'debug']( `browser[${level}] ${msg}` ); }, From 9f07f71500ca1871a0a0585b81268c028e60577d Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Wed, 13 Oct 2021 09:06:52 -0700 Subject: [PATCH 05/35] [build] Ironbank template updates (#114749) * [build] Ironbank template updates Signed-off-by: Tyler Smalley * Use placeholder Signed-off-by: Tyler Smalley --- .../docker_generator/bundle_dockerfiles.ts | 10 +- .../resources/ironbank/LICENSE | 280 ------------------ .../templates/ironbank/README.md | 4 +- .../ironbank/hardening_manifest.yaml | 4 + 4 files changed, 12 insertions(+), 286 deletions(-) delete mode 100644 src/dev/build/tasks/os_packages/docker_generator/resources/ironbank/LICENSE diff --git a/src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts b/src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts index 5f0665692b46f..02b469820f900 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts +++ b/src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts @@ -8,8 +8,9 @@ import { resolve } from 'path'; import { readFileSync } from 'fs'; +import { copyFile } from 'fs/promises'; -import { ToolingLog } from '@kbn/dev-utils'; +import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils'; import Mustache from 'mustache'; import { compressTar, copyAll, mkdirp, write, Config } from '../../../lib'; @@ -39,9 +40,10 @@ export async function bundleDockerFiles(config: Config, log: ToolingLog, scope: await copyAll(resolve(scope.dockerBuildDir, 'bin'), resolve(dockerFilesBuildDir, 'bin')); await copyAll(resolve(scope.dockerBuildDir, 'config'), resolve(dockerFilesBuildDir, 'config')); if (scope.ironbank) { - await copyAll(resolve(scope.dockerBuildDir), resolve(dockerFilesBuildDir), { - select: ['LICENSE'], - }); + await copyFile( + resolve(REPO_ROOT, 'licenses/ELASTIC-LICENSE-2.0.txt'), + resolve(dockerFilesBuildDir, 'LICENSE') + ); const templates = ['hardening_manifest.yaml', 'README.md']; for (const template of templates) { const file = readFileSync(resolve(__dirname, 'templates/ironbank', template)); diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/ironbank/LICENSE b/src/dev/build/tasks/os_packages/docker_generator/resources/ironbank/LICENSE deleted file mode 100644 index 632c3abe22e9b..0000000000000 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/ironbank/LICENSE +++ /dev/null @@ -1,280 +0,0 @@ -ELASTIC LICENSE AGREEMENT - -PLEASE READ CAREFULLY THIS ELASTIC LICENSE AGREEMENT (THIS "AGREEMENT"), WHICH -CONSTITUTES A LEGALLY BINDING AGREEMENT AND GOVERNS ALL OF YOUR USE OF ALL OF -THE ELASTIC SOFTWARE WITH WHICH THIS AGREEMENT IS INCLUDED ("ELASTIC SOFTWARE") -THAT IS PROVIDED IN OBJECT CODE FORMAT, AND, IN ACCORDANCE WITH SECTION 2 BELOW, -CERTAIN OF THE ELASTIC SOFTWARE THAT IS PROVIDED IN SOURCE CODE FORMAT. BY -INSTALLING OR USING ANY OF THE ELASTIC SOFTWARE GOVERNED BY THIS AGREEMENT, YOU -ARE ASSENTING TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE -WITH SUCH TERMS AND CONDITIONS, YOU MAY NOT INSTALL OR USE THE ELASTIC SOFTWARE -GOVERNED BY THIS AGREEMENT. IF YOU ARE INSTALLING OR USING THE SOFTWARE ON -BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL -AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF -SUCH ENTITY. - -Posted Date: April 20, 2018 - -This Agreement is entered into by and between Elasticsearch BV ("Elastic") and -You, or the legal entity on behalf of whom You are acting (as applicable, -"You"). - -1. OBJECT CODE END USER LICENSES, RESTRICTIONS AND THIRD PARTY OPEN SOURCE -SOFTWARE - - 1.1 Object Code End User License. Subject to the terms and conditions of - Section 1.2 of this Agreement, Elastic hereby grants to You, AT NO CHARGE and - for so long as you are not in breach of any provision of this Agreement, a - License to the Basic Features and Functions of the Elastic Software. - - 1.2 Reservation of Rights; Restrictions. As between Elastic and You, Elastic - and its licensors own all right, title and interest in and to the Elastic - Software, and except as expressly set forth in Sections 1.1, and 2.1 of this - Agreement, no other license to the Elastic Software is granted to You under - this Agreement, by implication, estoppel or otherwise. You agree not to: (i) - reverse engineer or decompile, decrypt, disassemble or otherwise reduce any - Elastic Software provided to You in Object Code, or any portion thereof, to - Source Code, except and only to the extent any such restriction is prohibited - by applicable law, (ii) except as expressly permitted in this Agreement, - prepare derivative works from, modify, copy or use the Elastic Software Object - Code or the Commercial Software Source Code in any manner; (iii) except as - expressly permitted in Section 1.1 above, transfer, sell, rent, lease, - distribute, sublicense, loan or otherwise transfer, Elastic Software Object - Code, in whole or in part, to any third party; (iv) use Elastic Software - Object Code for providing time-sharing services, any software-as-a-service, - service bureau services or as part of an application services provider or - other service offering (collectively, "SaaS Offering") where obtaining access - to the Elastic Software or the features and functions of the Elastic Software - is a primary reason or substantial motivation for users of the SaaS Offering - to access and/or use the SaaS Offering ("Prohibited SaaS Offering"); (v) - circumvent the limitations on use of Elastic Software provided to You in - Object Code format that are imposed or preserved by any License Key, or (vi) - alter or remove any Marks and Notices in the Elastic Software. If You have any - question as to whether a specific SaaS Offering constitutes a Prohibited SaaS - Offering, or are interested in obtaining Elastic's permission to engage in - commercial or non-commercial distribution of the Elastic Software, please - contact elastic_license@elastic.co. - - 1.3 Third Party Open Source Software. The Commercial Software may contain or - be provided with third party open source libraries, components, utilities and - other open source software (collectively, "Open Source Software"), which Open - Source Software may have applicable license terms as identified on a website - designated by Elastic. Notwithstanding anything to the contrary herein, use of - the Open Source Software shall be subject to the license terms and conditions - applicable to such Open Source Software, to the extent required by the - applicable licensor (which terms shall not restrict the license rights granted - to You hereunder, but may contain additional rights). To the extent any - condition of this Agreement conflicts with any license to the Open Source - Software, the Open Source Software license will govern with respect to such - Open Source Software only. Elastic may also separately provide you with - certain open source software that is licensed by Elastic. Your use of such - Elastic open source software will not be governed by this Agreement, but by - the applicable open source license terms. - -2. COMMERCIAL SOFTWARE SOURCE CODE - - 2.1 Limited License. Subject to the terms and conditions of Section 2.2 of - this Agreement, Elastic hereby grants to You, AT NO CHARGE and for so long as - you are not in breach of any provision of this Agreement, a limited, - non-exclusive, non-transferable, fully paid up royalty free right and license - to the Commercial Software in Source Code format, without the right to grant - or authorize sublicenses, to prepare Derivative Works of the Commercial - Software, provided You (i) do not hack the licensing mechanism, or otherwise - circumvent the intended limitations on the use of Elastic Software to enable - features other than Basic Features and Functions or those features You are - entitled to as part of a Subscription, and (ii) use the resulting object code - only for reasonable testing purposes. - - 2.2 Restrictions. Nothing in Section 2.1 grants You the right to (i) use the - Commercial Software Source Code other than in accordance with Section 2.1 - above, (ii) use a Derivative Work of the Commercial Software outside of a - Non-production Environment, in any production capacity, on a temporary or - permanent basis, or (iii) transfer, sell, rent, lease, distribute, sublicense, - loan or otherwise make available the Commercial Software Source Code, in whole - or in part, to any third party. Notwithstanding the foregoing, You may - maintain a copy of the repository in which the Source Code of the Commercial - Software resides and that copy may be publicly accessible, provided that you - include this Agreement with Your copy of the repository. - -3. TERMINATION - - 3.1 Termination. This Agreement will automatically terminate, whether or not - You receive notice of such Termination from Elastic, if You breach any of its - provisions. - - 3.2 Post Termination. Upon any termination of this Agreement, for any reason, - You shall promptly cease the use of the Elastic Software in Object Code format - and cease use of the Commercial Software in Source Code format. For the - avoidance of doubt, termination of this Agreement will not affect Your right - to use Elastic Software, in either Object Code or Source Code formats, made - available under the Apache License Version 2.0. - - 3.3 Survival. Sections 1.2, 2.2. 3.3, 4 and 5 shall survive any termination or - expiration of this Agreement. - -4. DISCLAIMER OF WARRANTIES AND LIMITATION OF LIABILITY - - 4.1 Disclaimer of Warranties. TO THE MAXIMUM EXTENT PERMITTED UNDER APPLICABLE - LAW, THE ELASTIC SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, - AND ELASTIC AND ITS LICENSORS MAKE NO WARRANTIES WHETHER EXPRESSED, IMPLIED OR - STATUTORY REGARDING OR RELATING TO THE ELASTIC SOFTWARE. TO THE MAXIMUM EXTENT - PERMITTED UNDER APPLICABLE LAW, ELASTIC AND ITS LICENSORS SPECIFICALLY - DISCLAIM ALL IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR - PURPOSE AND NON-INFRINGEMENT WITH RESPECT TO THE ELASTIC SOFTWARE, AND WITH - RESPECT TO THE USE OF THE FOREGOING. FURTHER, ELASTIC DOES NOT WARRANT RESULTS - OF USE OR THAT THE ELASTIC SOFTWARE WILL BE ERROR FREE OR THAT THE USE OF THE - ELASTIC SOFTWARE WILL BE UNINTERRUPTED. - - 4.2 Limitation of Liability. IN NO EVENT SHALL ELASTIC OR ITS LICENSORS BE - LIABLE TO YOU OR ANY THIRD PARTY FOR ANY DIRECT OR INDIRECT DAMAGES, - INCLUDING, WITHOUT LIMITATION, FOR ANY LOSS OF PROFITS, LOSS OF USE, BUSINESS - INTERRUPTION, LOSS OF DATA, COST OF SUBSTITUTE GOODS OR SERVICES, OR FOR ANY - SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, IN CONNECTION WITH - OR ARISING OUT OF THE USE OR INABILITY TO USE THE ELASTIC SOFTWARE, OR THE - PERFORMANCE OF OR FAILURE TO PERFORM THIS AGREEMENT, WHETHER ALLEGED AS A - BREACH OF CONTRACT OR TORTIOUS CONDUCT, INCLUDING NEGLIGENCE, EVEN IF ELASTIC - HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -5. MISCELLANEOUS - - This Agreement completely and exclusively states the entire agreement of the - parties regarding the subject matter herein, and it supersedes, and its terms - govern, all prior proposals, agreements, or other communications between the - parties, oral or written, regarding such subject matter. This Agreement may be - modified by Elastic from time to time, and any such modifications will be - effective upon the "Posted Date" set forth at the top of the modified - Agreement. If any provision hereof is held unenforceable, this Agreement will - continue without said provision and be interpreted to reflect the original - intent of the parties. This Agreement and any non-contractual obligation - arising out of or in connection with it, is governed exclusively by Dutch law. - This Agreement shall not be governed by the 1980 UN Convention on Contracts - for the International Sale of Goods. All disputes arising out of or in - connection with this Agreement, including its existence and validity, shall be - resolved by the courts with jurisdiction in Amsterdam, The Netherlands, except - where mandatory law provides for the courts at another location in The - Netherlands to have jurisdiction. The parties hereby irrevocably waive any and - all claims and defenses either might otherwise have in any such action or - proceeding in any of such courts based upon any alleged lack of personal - jurisdiction, improper venue, forum non conveniens or any similar claim or - defense. A breach or threatened breach, by You of Section 2 may cause - irreparable harm for which damages at law may not provide adequate relief, and - therefore Elastic shall be entitled to seek injunctive relief without being - required to post a bond. You may not assign this Agreement (including by - operation of law in connection with a merger or acquisition), in whole or in - part to any third party without the prior written consent of Elastic, which - may be withheld or granted by Elastic in its sole and absolute discretion. - Any assignment in violation of the preceding sentence is void. Notices to - Elastic may also be sent to legal@elastic.co. - -6. DEFINITIONS - - The following terms have the meanings ascribed: - - 6.1 "Affiliate" means, with respect to a party, any entity that controls, is - controlled by, or which is under common control with, such party, where - "control" means ownership of at least fifty percent (50%) of the outstanding - voting shares of the entity, or the contractual right to establish policy for, - and manage the operations of, the entity. - - 6.2 "Basic Features and Functions" means those features and functions of the - Elastic Software that are eligible for use under a Basic license, as set forth - at https://www.elastic.co/subscriptions, as may be modified by Elastic from - time to time. - - 6.3 "Commercial Software" means the Elastic Software Source Code in any file - containing a header stating the contents are subject to the Elastic License or - which is contained in the repository folder labeled "x-pack", unless a LICENSE - file present in the directory subtree declares a different license. - - 6.4 "Derivative Work of the Commercial Software" means, for purposes of this - Agreement, any modification(s) or enhancement(s) to the Commercial Software, - which represent, as a whole, an original work of authorship. - - 6.5 "License" means a limited, non-exclusive, non-transferable, fully paid up, - royalty free, right and license, without the right to grant or authorize - sublicenses, solely for Your internal business operations to (i) install and - use the applicable Features and Functions of the Elastic Software in Object - Code, and (ii) permit Contractors and Your Affiliates to use the Elastic - software as set forth in (i) above, provided that such use by Contractors must - be solely for Your benefit and/or the benefit of Your Affiliates, and You - shall be responsible for all acts and omissions of such Contractors and - Affiliates in connection with their use of the Elastic software that are - contrary to the terms and conditions of this Agreement. - - 6.6 "License Key" means a sequence of bytes, including but not limited to a - JSON blob, that is used to enable certain features and functions of the - Elastic Software. - - 6.7 "Marks and Notices" means all Elastic trademarks, trade names, logos and - notices present on the Documentation as originally provided by Elastic. - - 6.8 "Non-production Environment" means an environment for development, testing - or quality assurance, where software is not used for production purposes. - - 6.9 "Object Code" means any form resulting from mechanical transformation or - translation of Source Code form, including but not limited to compiled object - code, generated documentation, and conversions to other media types. - - 6.10 "Source Code" means the preferred form of computer software for making - modifications, including but not limited to software source code, - documentation source, and configuration files. - - 6.11 "Subscription" means the right to receive Support Services and a License - to the Commercial Software. - - -GOVERNMENT END USER ADDENDUM TO THE ELASTIC LICENSE AGREEMENT - - This ADDENDUM TO THE ELASTIC LICENSE AGREEMENT (this "Addendum") applies -only to U.S. Federal Government, State Government, and Local Government -entities ("Government End Users") of the Elastic Software. This Addendum is -subject to, and hereby incorporated into, the Elastic License Agreement, -which is being entered into as of even date herewith, by Elastic and You (the -"Agreement"). This Addendum sets forth additional terms and conditions -related to Your use of the Elastic Software. Capitalized terms not defined in -this Addendum have the meaning set forth in the Agreement. - - 1. LIMITED LICENSE TO DISTRIBUTE (DSOP ONLY). Subject to the terms and -conditions of the Agreement (including this Addendum), Elastic grants the -Department of Defense Enterprise DevSecOps Initiative (DSOP) a royalty-free, -non-exclusive, non-transferable, limited license to reproduce and distribute -the Elastic Software solely through a software distribution repository -controlled and managed by DSOP, provided that DSOP: (i) distributes the -Elastic Software complete and unmodified, inclusive of the Agreement -(including this Addendum) and (ii) does not remove or alter any proprietary -legends or notices contained in the Elastic Software. - - 2. CHOICE OF LAW. The choice of law and venue provisions set forth shall -prevail over those set forth in Section 5 of the Agreement. - - "For U.S. Federal Government Entity End Users. This Agreement and any - non-contractual obligation arising out of or in connection with it, is - governed exclusively by U.S. Federal law. To the extent permitted by - federal law, the laws of the State of Delaware (excluding Delaware choice - of law rules) will apply in the absence of applicable federal law. - - For State and Local Government Entity End Users. This Agreement and any - non-contractual obligation arising out of or in connection with it, is - governed exclusively by the laws of the state in which you are located - without reference to conflict of laws. Furthermore, the Parties agree that - the Uniform Computer Information Transactions Act or any version thereof, - adopted by any state in any form ('UCITA'), shall not apply to this - Agreement and, to the extent that UCITA is applicable, the Parties agree to - opt out of the applicability of UCITA pursuant to the opt-out provision(s) - contained therein." - - 3. ELASTIC LICENSE MODIFICATION. Section 5 of the Agreement is hereby -amended to replace - - "This Agreement may be modified by Elastic from time to time, and any - such modifications will be effective upon the "Posted Date" set forth at - the top of the modified Agreement." - - with: - - "This Agreement may be modified by Elastic from time to time; provided, - however, that any such modifications shall apply only to Elastic Software - that is installed after the "Posted Date" set forth at the top of the - modified Agreement." - -V100820.0 diff --git a/src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/README.md b/src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/README.md index d297d135149f4..d81b219900a85 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/README.md +++ b/src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/README.md @@ -31,9 +31,9 @@ You can learn more about the Elastic Community and also understand how to get mo visiting [Elastic Community](https://www.elastic.co/community). This software is governed by the [Elastic -License](https://github.com/elastic/elasticsearch/blob/{{branch}}/licenses/ELASTIC-LICENSE.txt), +License](https://github.com/elastic/kibana/blob/{{branch}}/licenses/ELASTIC-LICENSE-2.0.txt), and includes the full set of [free features](https://www.elastic.co/subscriptions). View the detailed release notes -[here](https://www.elastic.co/guide/en/elasticsearch/reference/{{branch}}/es-release-notes.html). +[here](https://www.elastic.co/guide/en/kibana/{{branch}}/release-notes-{{version}}.html). diff --git a/src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/hardening_manifest.yaml b/src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/hardening_manifest.yaml index 2e65e68bc2882..24614039e5eb7 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/hardening_manifest.yaml +++ b/src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/hardening_manifest.yaml @@ -56,3 +56,7 @@ maintainers: name: 'Alexander Klepal' username: 'alexander.klepal' cht_member: true + - email: "yalabe.dukuly@anchore.com" + name: "Yalabe Dukuly" + username: "yalabe.dukuly" + cht_member: true \ No newline at end of file From 3ab04b67f878cb126b986474bbfdaa84ad8abbc4 Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Wed, 13 Oct 2021 10:20:20 -0600 Subject: [PATCH 06/35] Allow users with read access to view Integrations app (#113925) --- .../public/applications/integrations/app.tsx | 77 ++-------- .../epm/screens/detail/assets/assets.tsx | 23 ++- .../epm/screens/detail/index.test.tsx | 13 ++ .../sections/epm/screens/detail/index.tsx | 143 ++++++++++++++---- .../fleet/public/components/header.tsx | 3 + .../fleet/public/hooks/use_request/app.ts | 9 +- x-pack/plugins/fleet/server/mocks/index.ts | 3 +- x-pack/plugins/fleet/server/plugin.ts | 23 ++- .../plugins/fleet/server/routes/app/index.ts | 27 ++-- .../plugins/fleet/server/routes/epm/index.ts | 15 +- .../plugins/fleet/server/routes/security.ts | 8 + .../fleet/server/services/app_context.ts | 21 ++- .../server/services/epm/archive/cache.ts | 8 + .../server/services/epm/archive/index.ts | 4 + .../plugins/fleet/storybook/context/http.ts | 4 + .../translations/translations/ja-JP.json | 3 - .../translations/translations/zh-CN.json | 3 - .../fleet_api_integration/apis/agents/list.ts | 57 +------ .../apis/epm/bulk_upgrade.ts | 9 ++ .../fleet_api_integration/apis/epm/delete.ts | 10 ++ .../fleet_api_integration/apis/epm/get.ts | 9 ++ .../apis/epm/install_by_upload.ts | 13 ++ .../fleet_api_integration/apis/epm/list.ts | 9 ++ .../test/fleet_api_integration/apis/index.js | 10 +- .../fleet_api_integration/apis/test_users.ts | 63 ++++++++ 25 files changed, 370 insertions(+), 197 deletions(-) create mode 100644 x-pack/test/fleet_api_integration/apis/test_users.ts diff --git a/x-pack/plugins/fleet/public/applications/integrations/app.tsx b/x-pack/plugins/fleet/public/applications/integrations/app.tsx index b10cef9d3ffe4..771b17ae8c3ee 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/app.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/app.tsx @@ -7,12 +7,10 @@ import React, { memo, useEffect, useState } from 'react'; import type { AppMountParameters } from 'kibana/public'; -import { EuiCode, EuiEmptyPrompt, EuiErrorBoundary, EuiPanel, EuiPortal } from '@elastic/eui'; +import { EuiErrorBoundary, EuiPortal } from '@elastic/eui'; import type { History } from 'history'; import { Router, Redirect, Route, Switch } from 'react-router-dom'; import { FormattedMessage } from '@kbn/i18n/react'; -import { i18n } from '@kbn/i18n'; -import styled from 'styled-components'; import useObservable from 'react-use/lib/useObservable'; import { @@ -49,29 +47,23 @@ const ErrorLayout = ({ children }: { children: JSX.Element }) => ( ); -const Panel = styled(EuiPanel)` - max-width: 500px; - margin-right: auto; - margin-left: auto; -`; - export const WithPermissionsAndSetup: React.FC = memo(({ children }) => { useBreadcrumbs('integrations'); const [isPermissionsLoading, setIsPermissionsLoading] = useState(false); - const [permissionsError, setPermissionsError] = useState(); const [isInitialized, setIsInitialized] = useState(false); const [initializationError, setInitializationError] = useState(null); useEffect(() => { (async () => { - setPermissionsError(undefined); setIsInitialized(false); setInitializationError(null); try { + // Attempt Fleet Setup if user has permissions, otherwise skip setIsPermissionsLoading(true); const permissionsResponse = await sendGetPermissionsCheck(); setIsPermissionsLoading(false); + if (permissionsResponse.data?.success) { try { const setupResponse = await sendSetup(); @@ -83,69 +75,20 @@ export const WithPermissionsAndSetup: React.FC = memo(({ children }) => { } setIsInitialized(true); } else { - setPermissionsError(permissionsResponse.data?.error || 'REQUEST_ERROR'); + setIsInitialized(true); } - } catch (err) { - setPermissionsError('REQUEST_ERROR'); + } catch { + // If there's an error checking permissions, default to proceeding without running setup + // User will only have access to EPM endpoints if they actually have permission + setIsInitialized(true); } })(); }, []); - if (isPermissionsLoading || permissionsError) { + if (isPermissionsLoading) { return ( - {isPermissionsLoading ? ( - - ) : permissionsError === 'REQUEST_ERROR' ? ( - - } - error={i18n.translate('xpack.fleet.permissionsRequestErrorMessageDescription', { - defaultMessage: 'There was a problem checking Fleet permissions', - })} - /> - ) : ( - - - {permissionsError === 'MISSING_SUPERUSER_ROLE' ? ( - - ) : ( - - )} - - } - body={ -

- {permissionsError === 'MISSING_SUPERUSER_ROLE' ? ( - superuser }} - /> - ) : ( - - )} -

- } - /> - - )} + ); } diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx index 6d075faeef308..a8d27580e0bd1 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx @@ -9,6 +9,7 @@ import React, { useEffect, useState } from 'react'; import { Redirect } from 'react-router-dom'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, EuiFlexItem, EuiTitle, EuiSpacer } from '@elastic/eui'; +import { groupBy } from 'lodash'; import { Loading, Error, ExtensionWrapper } from '../../../../../components'; @@ -67,8 +68,26 @@ export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => { id, type, })); - const { savedObjects } = await savedObjectsClient.bulkGet(objectsToGet); - setAssetsSavedObjects(savedObjects as AssetSavedObject[]); + + // We don't have an API to know which SO types a user has access to, so instead we make a request for each + // SO type and ignore the 403 errors + const objectsByType = await Promise.all( + Object.entries(groupBy(objectsToGet, 'type')).map(([type, objects]) => + savedObjectsClient + .bulkGet(objects) + // Ignore privilege errors + .catch((e: any) => { + if (e?.body?.statusCode === 403) { + return { savedObjects: [] }; + } else { + throw e; + } + }) + .then(({ savedObjects }) => savedObjects as AssetSavedObject[]) + ) + ); + + setAssetsSavedObjects(objectsByType.flat()); } catch (e) { setFetchError(e); } finally { diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/index.test.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/index.test.tsx index d70b6c68016be..d442f8a13e27e 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/index.test.tsx @@ -11,6 +11,7 @@ import { act, cleanup } from '@testing-library/react'; import { INTEGRATIONS_ROUTING_PATHS, pagePathGetters } from '../../../../constants'; import type { + CheckPermissionsResponse, GetAgentPoliciesResponse, GetFleetStatusResponse, GetInfoResponse, @@ -23,6 +24,7 @@ import type { } from '../../../../../../../common/types/models'; import { agentPolicyRouteService, + appRoutesService, epmRouteService, fleetSetupRouteService, packagePolicyRouteService, @@ -260,6 +262,7 @@ interface EpmPackageDetailsResponseProvidersMock { fleetSetup: jest.MockedFunction<() => GetFleetStatusResponse>; packagePolicyList: jest.MockedFunction<() => GetPackagePoliciesResponse>; agentPolicyList: jest.MockedFunction<() => GetAgentPoliciesResponse>; + appCheckPermissions: jest.MockedFunction<() => CheckPermissionsResponse>; } const mockApiCalls = ( @@ -740,6 +743,10 @@ On Windows, the module was tested with Nginx installed from the Chocolatey repos }, }; + const appCheckPermissionsResponse: CheckPermissionsResponse = { + success: true, + }; + const mockedApiInterface: MockedApi = { waitForApi() { return new Promise((resolve) => { @@ -757,6 +764,7 @@ On Windows, the module was tested with Nginx installed from the Chocolatey repos fleetSetup: jest.fn().mockReturnValue(agentsSetupResponse), packagePolicyList: jest.fn().mockReturnValue(packagePoliciesResponse), agentPolicyList: jest.fn().mockReturnValue(agentPoliciesResponse), + appCheckPermissions: jest.fn().mockReturnValue(appCheckPermissionsResponse), }, }; @@ -792,6 +800,11 @@ On Windows, the module was tested with Nginx installed from the Chocolatey repos return mockedApiInterface.responseProvider.epmGetStats(); } + if (path === appRoutesService.getCheckPermissionsPath()) { + markApiCallAsHandled(); + return mockedApiInterface.responseProvider.appCheckPermissions(); + } + const err = new Error(`API [GET ${path}] is not MOCKED!`); // eslint-disable-next-line no-console console.error(err); diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/index.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/index.tsx index 82436eb4d3f51..ade290aab4e5e 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/index.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/index.tsx @@ -8,10 +8,12 @@ import type { ReactEventHandler } from 'react'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { Redirect, Route, Switch, useLocation, useParams, useHistory } from 'react-router-dom'; import styled from 'styled-components'; +import type { EuiToolTipProps } from '@elastic/eui'; import { EuiBetaBadge, EuiButton, EuiButtonEmpty, + EuiCallOut, EuiDescriptionList, EuiDescriptionListDescription, EuiDescriptionListTitle, @@ -19,6 +21,7 @@ import { EuiFlexItem, EuiSpacer, EuiText, + EuiToolTip, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -30,6 +33,7 @@ import { useUIExtension, useBreadcrumbs, useStartServices, + usePermissionCheck, } from '../../../../hooks'; import { PLUGIN_ID, @@ -61,6 +65,7 @@ import { OverviewPage } from './overview'; import { PackagePoliciesPage } from './policies'; import { SettingsPage } from './settings'; import { CustomViewPage } from './custom'; + import './index.scss'; export interface DetailParams { @@ -95,7 +100,11 @@ export function Detail() { const { getId: getAgentPolicyId } = useAgentPolicyContext(); const { pkgkey, panel } = useParams(); const { getHref } = useLink(); - const hasWriteCapabilites = useCapabilities().write; + const hasWriteCapabilities = useCapabilities().write; + const permissionCheck = usePermissionCheck(); + const missingSecurityConfiguration = + !permissionCheck.data?.success && permissionCheck.data?.error === 'MISSING_SECURITY'; + const userCanInstallIntegrations = hasWriteCapabilities && permissionCheck.data?.success; const history = useHistory(); const { pathname, search, hash } = useLocation(); const queryParams = useMemo(() => new URLSearchParams(search), [search]); @@ -127,9 +136,11 @@ export function Detail() { const { data: packageInfoData, error: packageInfoError, - isLoading, + isLoading: packageInfoLoading, } = useGetPackageInfoByKey(pkgkey); + const isLoading = packageInfoLoading || permissionCheck.isLoading; + const showCustomTab = useUIExtension(packageInfoData?.response.name ?? '', 'package-detail-custom') !== undefined; @@ -327,10 +338,9 @@ export function Detail() { { isDivider: true }, { content: ( - // eslint-disable-next-line @elastic/eui/href-or-on-click - + ) : ( + + ), + } + : undefined + } > - + ), }, ].map((item, index) => ( @@ -370,16 +397,17 @@ export function Detail() { ) : undefined, [ - getHref, - handleAddIntegrationPolicyClick, - hasWriteCapabilites, - integration, - integrationInfo, packageInfo, + updateAvailable, packageInstallStatus, + userCanInstallIntegrations, + getHref, pkgkey, - updateAvailable, + integration, agentPolicyIdFromContext, + handleAddIntegrationPolicyClick, + missingSecurityConfiguration, + integrationInfo?.title, ] ); @@ -407,7 +435,7 @@ export function Detail() { }, ]; - if (packageInstallStatus === InstallStatus.installed) { + if (userCanInstallIntegrations && packageInstallStatus === InstallStatus.installed) { tabs.push({ id: 'policies', name: ( @@ -443,21 +471,23 @@ export function Detail() { }); } - tabs.push({ - id: 'settings', - name: ( - - ), - isSelected: panel === 'settings', - 'data-test-subj': `tab-settings`, - href: getHref('integration_details_settings', { - pkgkey: packageInfoKey, - ...(integration ? { integration } : {}), - }), - }); + if (userCanInstallIntegrations) { + tabs.push({ + id: 'settings', + name: ( + + ), + isSelected: panel === 'settings', + 'data-test-subj': `tab-settings`, + href: getHref('integration_details_settings', { + pkgkey: packageInfoKey, + ...(integration ? { integration } : {}), + }), + }); + } if (showCustomTab) { tabs.push({ @@ -478,13 +508,55 @@ export function Detail() { } return tabs; - }, [packageInfo, panel, getHref, integration, packageInstallStatus, showCustomTab, CustomAssets]); + }, [ + packageInfo, + panel, + getHref, + integration, + userCanInstallIntegrations, + packageInstallStatus, + CustomAssets, + showCustomTab, + ]); + + const securityCallout = missingSecurityConfiguration ? ( + <> + + } + > + + + + ), + }} + /> + + + + ) : undefined; return ( @@ -526,3 +598,16 @@ export function Detail() { ); } + +type EuiButtonPropsFull = Parameters[0]; + +const EuiButtonWithTooltip: React.FC }> = + ({ tooltip: tooltipProps, ...buttonProps }) => { + return tooltipProps ? ( + + + + ) : ( + + ); + }; diff --git a/x-pack/plugins/fleet/public/components/header.tsx b/x-pack/plugins/fleet/public/components/header.tsx index 80cebe3b0b304..2a8b20240a4f6 100644 --- a/x-pack/plugins/fleet/public/components/header.tsx +++ b/x-pack/plugins/fleet/public/components/header.tsx @@ -43,6 +43,7 @@ export interface HeaderProps { leftColumn?: JSX.Element; rightColumn?: JSX.Element; rightColumnGrow?: EuiFlexItemProps['grow']; + topContent?: JSX.Element; tabs?: Array & { name?: JSX.Element | string }>; tabsClassName?: string; 'data-test-subj'?: string; @@ -61,6 +62,7 @@ export const Header: React.FC = ({ leftColumn, rightColumn, rightColumnGrow, + topContent, tabs, maxWidth, tabsClassName, @@ -68,6 +70,7 @@ export const Header: React.FC = ({ }) => ( + {topContent} { return sendRequest({ @@ -23,3 +23,10 @@ export const sendGenerateServiceToken = () => { method: 'post', }); }; + +export const usePermissionCheck = () => { + return useRequest({ + path: appRoutesService.getCheckPermissionsPath(), + method: 'get', + }); +}; diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 43b455045e72b..0e7b335da6775 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -37,7 +37,8 @@ export const createAppContextStartContractMock = (): FleetAppContext => { data: dataPluginMock.createStartContract(), encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(), savedObjects: savedObjectsServiceMock.createStartContract(), - security: securityMock.createStart(), + securitySetup: securityMock.createSetup(), + securityStart: securityMock.createStart(), logger: loggingSystemMock.create().get(), isProductionMode: true, configInitialValue: { diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 6aad028666ee8..a706ca6a54fdc 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -37,6 +37,7 @@ import { AGENT_POLICY_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE, + ASSETS_SAVED_OBJECT_TYPE, AGENT_SAVED_OBJECT_TYPE, ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, @@ -103,7 +104,8 @@ export interface FleetAppContext { data: DataPluginStart; encryptedSavedObjectsStart?: EncryptedSavedObjectsPluginStart; encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup; - security?: SecurityPluginStart; + securitySetup?: SecurityPluginSetup; + securityStart?: SecurityPluginStart; config$?: Observable; configInitialValue: FleetConfigType; savedObjects: SavedObjectsServiceStart; @@ -122,6 +124,7 @@ const allSavedObjectTypes = [ AGENT_POLICY_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE, + ASSETS_SAVED_OBJECT_TYPE, AGENT_SAVED_OBJECT_TYPE, ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, @@ -164,14 +167,15 @@ export class FleetPlugin private licensing$!: Observable; private config$: Observable; private configInitialValue: FleetConfigType; - private cloud: CloudSetup | undefined; - private logger: Logger | undefined; + private cloud?: CloudSetup; + private logger?: Logger; private isProductionMode: FleetAppContext['isProductionMode']; private kibanaVersion: FleetAppContext['kibanaVersion']; private kibanaBranch: FleetAppContext['kibanaBranch']; - private httpSetup: HttpServiceSetup | undefined; - private encryptedSavedObjectsSetup: EncryptedSavedObjectsPluginSetup | undefined; + private httpSetup?: HttpServiceSetup; + private securitySetup?: SecurityPluginSetup; + private encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup; constructor(private readonly initializerContext: PluginInitializerContext) { this.config$ = this.initializerContext.config.create(); @@ -187,6 +191,7 @@ export class FleetPlugin this.licensing$ = deps.licensing.license$; this.encryptedSavedObjectsSetup = deps.encryptedSavedObjects; this.cloud = deps.cloud; + this.securitySetup = deps.security; const config = this.configInitialValue; registerSavedObjects(core.savedObjects, deps.encryptedSavedObjects); @@ -233,6 +238,10 @@ export class FleetPlugin // Always register app routes for permissions checking registerAppRoutes(router); + // Allow read-only users access to endpoints necessary for Integrations UI + // Only some endpoints require superuser so we pass a raw IRouter here + registerEPMRoutes(router); + // For all the routes we enforce the user to have role superuser const routerSuperuserOnly = makeRouterEnforcingSuperuser(router); // Register rest of routes only if security is enabled @@ -243,7 +252,6 @@ export class FleetPlugin registerOutputRoutes(routerSuperuserOnly); registerSettingsRoutes(routerSuperuserOnly); registerDataStreamRoutes(routerSuperuserOnly); - registerEPMRoutes(routerSuperuserOnly); registerPreconfigurationRoutes(routerSuperuserOnly); // Conditional config routes @@ -260,7 +268,8 @@ export class FleetPlugin data: plugins.data, encryptedSavedObjectsStart: plugins.encryptedSavedObjects, encryptedSavedObjectsSetup: this.encryptedSavedObjectsSetup, - security: plugins.security, + securitySetup: this.securitySetup, + securityStart: plugins.security, configInitialValue: this.configInitialValue, config$: this.config$, savedObjects: core.savedObjects, diff --git a/x-pack/plugins/fleet/server/routes/app/index.ts b/x-pack/plugins/fleet/server/routes/app/index.ts index 025da6d79702c..43614f3a286b0 100644 --- a/x-pack/plugins/fleet/server/routes/app/index.ts +++ b/x-pack/plugins/fleet/server/routes/app/index.ts @@ -17,13 +17,15 @@ export const getCheckPermissionsHandler: RequestHandler = async (context, reques success: false, error: 'MISSING_SECURITY', }; - const body: CheckPermissionsResponse = { success: true }; - try { - const security = await appContextService.getSecurity(); + + if (!appContextService.hasSecurity() || !appContextService.getSecurityLicense().isEnabled()) { + return response.ok({ body: missingSecurityBody }); + } else { + const security = appContextService.getSecurity(); const user = security.authc.getCurrentUser(request); - // when ES security is disabled, but Kibana security plugin is not explicitly disabled, - // `authc.getCurrentUser()` does not error, instead it comes back as `null` + // Defensively handle situation where user is undefined (should only happen when ES security is disabled) + // This should be covered by the `getSecurityLicense().isEnabled()` check above, but we leave this for robustness. if (!user) { return response.ok({ body: missingSecurityBody, @@ -31,20 +33,15 @@ export const getCheckPermissionsHandler: RequestHandler = async (context, reques } if (!user?.roles.includes('superuser')) { - body.success = false; - body.error = 'MISSING_SUPERUSER_ROLE'; return response.ok({ - body, + body: { + success: false, + error: 'MISSING_SUPERUSER_ROLE', + } as CheckPermissionsResponse, }); } - return response.ok({ body: { success: true } }); - } catch (e) { - // when Kibana security plugin is explicitly disabled, - // `appContextService.getSecurity()` returns an error, so we catch it here - return response.ok({ - body: missingSecurityBody, - }); + return response.ok({ body: { success: true } as CheckPermissionsResponse }); } }; diff --git a/x-pack/plugins/fleet/server/routes/epm/index.ts b/x-pack/plugins/fleet/server/routes/epm/index.ts index 684547dc1862c..0f49b3cfa772d 100644 --- a/x-pack/plugins/fleet/server/routes/epm/index.ts +++ b/x-pack/plugins/fleet/server/routes/epm/index.ts @@ -20,6 +20,7 @@ import { GetStatsRequestSchema, UpdatePackageRequestSchema, } from '../../types'; +import { enforceSuperUser } from '../security'; import { getCategoriesHandler, @@ -60,7 +61,7 @@ export const registerRoutes = (router: IRouter) => { { path: EPM_API_ROUTES.LIMITED_LIST_PATTERN, validate: false, - options: { tags: [`access:${PLUGIN_ID}`] }, + options: { tags: [`access:${PLUGIN_ID}-read`] }, }, getLimitedListHandler ); @@ -69,7 +70,7 @@ export const registerRoutes = (router: IRouter) => { { path: EPM_API_ROUTES.STATS_PATTERN, validate: GetStatsRequestSchema, - options: { tags: [`access:${PLUGIN_ID}`] }, + options: { tags: [`access:${PLUGIN_ID}-read`] }, }, getStatsHandler ); @@ -98,7 +99,7 @@ export const registerRoutes = (router: IRouter) => { validate: UpdatePackageRequestSchema, options: { tags: [`access:${PLUGIN_ID}-all`] }, }, - updatePackageHandler + enforceSuperUser(updatePackageHandler) ); router.post( @@ -107,7 +108,7 @@ export const registerRoutes = (router: IRouter) => { validate: InstallPackageFromRegistryRequestSchema, options: { tags: [`access:${PLUGIN_ID}-all`] }, }, - installPackageFromRegistryHandler + enforceSuperUser(installPackageFromRegistryHandler) ); router.post( @@ -116,7 +117,7 @@ export const registerRoutes = (router: IRouter) => { validate: BulkUpgradePackagesFromRegistryRequestSchema, options: { tags: [`access:${PLUGIN_ID}-all`] }, }, - bulkInstallPackagesFromRegistryHandler + enforceSuperUser(bulkInstallPackagesFromRegistryHandler) ); router.post( @@ -132,7 +133,7 @@ export const registerRoutes = (router: IRouter) => { }, }, }, - installPackageByUploadHandler + enforceSuperUser(installPackageByUploadHandler) ); router.delete( @@ -141,6 +142,6 @@ export const registerRoutes = (router: IRouter) => { validate: DeletePackageRequestSchema, options: { tags: [`access:${PLUGIN_ID}-all`] }, }, - deletePackageHandler + enforceSuperUser(deletePackageHandler) ); }; diff --git a/x-pack/plugins/fleet/server/routes/security.ts b/x-pack/plugins/fleet/server/routes/security.ts index 60011dcf3d33f..8efea34ed8164 100644 --- a/x-pack/plugins/fleet/server/routes/security.ts +++ b/x-pack/plugins/fleet/server/routes/security.ts @@ -13,6 +13,14 @@ export function enforceSuperUser( handler: RequestHandler ): RequestHandler { return function enforceSuperHandler(context, req, res) { + if (!appContextService.hasSecurity() || !appContextService.getSecurityLicense().isEnabled()) { + return res.forbidden({ + body: { + message: `Access to this API requires that security is enabled`, + }, + }); + } + const security = appContextService.getSecurity(); const user = security.authc.getCurrentUser(req); if (!user) { diff --git a/x-pack/plugins/fleet/server/services/app_context.ts b/x-pack/plugins/fleet/server/services/app_context.ts index 1fb34a9a399eb..a1e6ef4545aef 100644 --- a/x-pack/plugins/fleet/server/services/app_context.ts +++ b/x-pack/plugins/fleet/server/services/app_context.ts @@ -22,7 +22,7 @@ import type { EncryptedSavedObjectsPluginSetup, } from '../../../encrypted_saved_objects/server'; -import type { SecurityPluginStart } from '../../../security/server'; +import type { SecurityPluginStart, SecurityPluginSetup } from '../../../security/server'; import type { FleetConfigType } from '../../common'; import type { ExternalCallback, @@ -39,7 +39,8 @@ class AppContextService { private encryptedSavedObjectsSetup: EncryptedSavedObjectsPluginSetup | undefined; private data: DataPluginStart | undefined; private esClient: ElasticsearchClient | undefined; - private security: SecurityPluginStart | undefined; + private securitySetup: SecurityPluginSetup | undefined; + private securityStart: SecurityPluginStart | undefined; private config$?: Observable; private configSubject$?: BehaviorSubject; private savedObjects: SavedObjectsServiceStart | undefined; @@ -56,7 +57,8 @@ class AppContextService { this.esClient = appContext.elasticsearch.client.asInternalUser; this.encryptedSavedObjects = appContext.encryptedSavedObjectsStart?.getClient(); this.encryptedSavedObjectsSetup = appContext.encryptedSavedObjectsSetup; - this.security = appContext.security; + this.securitySetup = appContext.securitySetup; + this.securityStart = appContext.securityStart; this.savedObjects = appContext.savedObjects; this.isProductionMode = appContext.isProductionMode; this.cloud = appContext.cloud; @@ -92,14 +94,21 @@ class AppContextService { } public getSecurity() { - if (!this.security) { + if (!this.hasSecurity()) { throw new Error('Security service not set.'); } - return this.security; + return this.securityStart!; + } + + public getSecurityLicense() { + if (!this.hasSecurity()) { + throw new Error('Security service not set.'); + } + return this.securitySetup!.license; } public hasSecurity() { - return !!this.security; + return !!this.securitySetup && !!this.securityStart; } public getCloud() { diff --git a/x-pack/plugins/fleet/server/services/epm/archive/cache.ts b/x-pack/plugins/fleet/server/services/epm/archive/cache.ts index 7f479dc5d6b63..676c3aa571a0f 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/cache.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/cache.ts @@ -67,3 +67,11 @@ export const setPackageInfo = ({ }; export const deletePackageInfo = (args: SharedKey) => packageInfoCache.delete(sharedKey(args)); + +export const clearPackageFileCache = (args: SharedKey) => { + const fileList = getArchiveFilelist(args) ?? []; + fileList.forEach((filePath) => { + deleteArchiveEntry(filePath); + }); + deleteArchiveFilelist(args); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.ts index b08ec815a394d..7c590eb1bcd39 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/index.ts @@ -16,6 +16,7 @@ import { setArchiveFilelist, deleteArchiveFilelist, deletePackageInfo, + clearPackageFileCache, } from './cache'; import type { SharedKey } from './cache'; import { getBufferExtractor } from './extract'; @@ -42,6 +43,9 @@ export async function unpackBufferToCache({ archiveBuffer: Buffer; installSource: InstallSource; }): Promise { + // Make sure any buffers from previous installations from registry or upload are deleted first + clearPackageFileCache({ name, version }); + const entries = await unpackBufferEntries(archiveBuffer, contentType); const paths: string[] = []; entries.forEach((entry) => { diff --git a/x-pack/plugins/fleet/storybook/context/http.ts b/x-pack/plugins/fleet/storybook/context/http.ts index f62970c0c0306..3e515c075a595 100644 --- a/x-pack/plugins/fleet/storybook/context/http.ts +++ b/x-pack/plugins/fleet/storybook/context/http.ts @@ -75,6 +75,10 @@ export const getHttp = (basepath = BASE_PATH) => { return await import('./fixtures/integration.okta'); } + if (path.startsWith('/api/fleet/check-permissions')) { + return { success: true }; + } + action(path)('KP: UNSUPPORTED ROUTE'); return {}; }) as HttpHandler, diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 2c828513e9d33..482a493865a47 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -10873,8 +10873,6 @@ "xpack.fleet.integrations.updatePackage.updatePackageButtonLabel": "最新バージョンに更新", "xpack.fleet.integrationsAppTitle": "統合", "xpack.fleet.integrationsHeaderTitle": "Elasticエージェント統合", - "xpack.fleet.integrationsPermissionDeniedErrorMessage": "統合にアクセスする権限がありません。統合には{roleName}権限が必要です。", - "xpack.fleet.integrationsSecurityRequiredErrorMessage": "統合を使用するには、KibanaとElasticsearchでセキュリティを有効にする必要があります。", "xpack.fleet.invalidLicenseDescription": "現在のライセンスは期限切れです。登録されたビートエージェントは引き続き動作しますが、Elastic Fleet インターフェイスにアクセスするには有効なライセンスが必要です。", "xpack.fleet.invalidLicenseTitle": "ライセンスの期限切れ", "xpack.fleet.multiTextInput.addRow": "行の追加", @@ -10946,7 +10944,6 @@ "xpack.fleet.preconfiguration.missingIDError": "{agentPolicyName}には「id」フィールドがありません。ポリシーのis_defaultまたはis_default_fleet_serverに設定されている場合をのぞき、「id」は必須です。", "xpack.fleet.preconfiguration.packageMissingError": "{agentPolicyName}を追加できませんでした。{pkgName}がインストールされていません。{pkgName}を`{packagesConfigValue}`に追加するか、{packagePolicyName}から削除してください。", "xpack.fleet.preconfiguration.policyDeleted": "構成済みのポリシー{id}が削除されました。作成をスキップしています", - "xpack.fleet.securityRequiredErrorTitle": "セキュリティが有効ではありません", "xpack.fleet.serverError.agentPolicyDoesNotExist": "エージェントポリシー{agentPolicyId}が存在しません", "xpack.fleet.serverError.enrollmentKeyDuplicate": "エージェントポリシーの{agentPolicyId}登録キー{providedKeyName}はすでに存在します", "xpack.fleet.serverError.returnedIncorrectKey": "find enrollmentKeyByIdで正しくないキーが返されました", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index c0d3975461c7b..2322f6213149f 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -10987,8 +10987,6 @@ "xpack.fleet.integrations.updatePackage.updatePackageButtonLabel": "更新到最新版本", "xpack.fleet.integrationsAppTitle": "集成", "xpack.fleet.integrationsHeaderTitle": "Elastic 代理集成", - "xpack.fleet.integrationsPermissionDeniedErrorMessage": "您无权访问“集成”。“集成”需要 {roleName} 权限。", - "xpack.fleet.integrationsSecurityRequiredErrorMessage": "必须在 Kibana 和 Elasticsearch 中启用安全性，才能使用“集成”。", "xpack.fleet.invalidLicenseDescription": "您当前的许可证已过期。已注册 Beats 代理将继续工作，但您需要有效的许可证，才能访问 Elastic Fleet 界面。", "xpack.fleet.invalidLicenseTitle": "已过期许可证", "xpack.fleet.multiTextInput.addRow": "添加行", @@ -11060,7 +11058,6 @@ "xpack.fleet.preconfiguration.missingIDError": "{agentPolicyName} 缺失 `id` 字段。`id` 是必需的，但标记为 is_default 或 is_default_fleet_server 的策略除外。", "xpack.fleet.preconfiguration.packageMissingError": "{agentPolicyName} 无法添加。{pkgName} 未安装，请将 {pkgName} 添加到 `{packagesConfigValue}` 或将其从 {packagePolicyName} 中移除。", "xpack.fleet.preconfiguration.policyDeleted": "预配置的策略 {id} 已删除；将跳过创建", - "xpack.fleet.securityRequiredErrorTitle": "安全性未启用", "xpack.fleet.serverError.agentPolicyDoesNotExist": "代理策略 {agentPolicyId} 不存在", "xpack.fleet.serverError.enrollmentKeyDuplicate": "称作 {providedKeyName} 的注册密钥对于代理策略 {agentPolicyId} 已存在", "xpack.fleet.serverError.returnedIncorrectKey": "find enrollmentKeyById 返回错误的密钥", diff --git a/x-pack/test/fleet_api_integration/apis/agents/list.ts b/x-pack/test/fleet_api_integration/apis/agents/list.ts index a11f4d49fe0f1..3795734f60fe0 100644 --- a/x-pack/test/fleet_api_integration/apis/agents/list.ts +++ b/x-pack/test/fleet_api_integration/apis/agents/list.ts @@ -8,66 +8,15 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { testUsers } from '../test_users'; export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const supertestWithoutAuth = getService('supertestWithoutAuth'); const supertest = getService('supertest'); - const security = getService('security'); - const users: { [rollName: string]: { username: string; password: string; permissions?: any } } = { - kibana_basic_user: { - permissions: { - feature: { - dashboards: ['read'], - }, - spaces: ['*'], - }, - username: 'kibana_basic_user', - password: 'changeme', - }, - fleet_user: { - permissions: { - feature: { - fleet: ['read'], - }, - spaces: ['*'], - }, - username: 'fleet_user', - password: 'changeme', - }, - fleet_admin: { - permissions: { - feature: { - fleet: ['all'], - }, - spaces: ['*'], - }, - username: 'fleet_admin', - password: 'changeme', - }, - }; describe('fleet_list_agent', () => { before(async () => { - for (const roleName in users) { - if (users.hasOwnProperty(roleName)) { - const user = users[roleName]; - - if (user.permissions) { - await security.role.create(roleName, { - kibana: [user.permissions], - }); - } - - // Import a repository first - await security.user.create(user.username, { - password: user.password, - roles: [roleName], - full_name: user.username, - }); - } - } - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/fleet/agents'); }); after(async () => { @@ -77,13 +26,13 @@ export default function ({ getService }: FtrProviderContext) { it('should return a 403 if a user without the superuser role try to access the APU', async () => { await supertestWithoutAuth .get(`/api/fleet/agents`) - .auth(users.fleet_admin.username, users.fleet_admin.password) + .auth(testUsers.fleet_all.username, testUsers.fleet_all.password) .expect(403); }); it('should not return the list of agents when requesting as a user without fleet permissions', async () => { await supertestWithoutAuth .get(`/api/fleet/agents`) - .auth(users.kibana_basic_user.username, users.kibana_basic_user.password) + .auth(testUsers.fleet_no_access.username, testUsers.fleet_no_access.password) .expect(403); }); diff --git a/x-pack/test/fleet_api_integration/apis/epm/bulk_upgrade.ts b/x-pack/test/fleet_api_integration/apis/epm/bulk_upgrade.ts index bb75629e222a5..3b3ccb03e56f3 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/bulk_upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/bulk_upgrade.ts @@ -14,10 +14,12 @@ import { IBulkInstallPackageHTTPError, } from '../../../../plugins/fleet/common'; import { setupFleetAndAgents } from '../agents/services'; +import { testUsers } from '../test_users'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); const deletePackage = async (pkgkey: string) => { await supertest.delete(`/api/fleet/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx'); @@ -44,6 +46,13 @@ export default function (providerContext: FtrProviderContext) { it('should return 400 if no packages are requested for upgrade', async function () { await supertest.post(`/api/fleet/epm/packages/_bulk`).set('kbn-xsrf', 'xxxx').expect(400); }); + it('should return 403 if read only user requests upgrade', async function () { + await supertestWithoutAuth + .post(`/api/fleet/epm/packages/_bulk`) + .auth(testUsers.fleet_read_only.username, testUsers.fleet_read_only.password) + .set('kbn-xsrf', 'xxxx') + .expect(403); + }); it('should return 200 and an array for upgrading a package', async function () { const { body }: { body: BulkInstallPackagesResponse } = await supertest .post(`/api/fleet/epm/packages/_bulk`) diff --git a/x-pack/test/fleet_api_integration/apis/epm/delete.ts b/x-pack/test/fleet_api_integration/apis/epm/delete.ts index 40650c4c176f7..9980d85ac171e 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/delete.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/delete.ts @@ -8,10 +8,12 @@ import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { setupFleetAndAgents } from '../agents/services'; +import { testUsers } from '../test_users'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); const requiredPackage = 'elastic_agent-0.0.7'; const installPackage = async (pkgkey: string) => { @@ -52,5 +54,13 @@ export default function (providerContext: FtrProviderContext) { .send({ force: true }) .expect(200); }); + + it('should return 403 for read-only users', async () => { + await supertestWithoutAuth + .delete(`/api/fleet/epm/packages/${requiredPackage}`) + .auth(testUsers.fleet_read_only.username, testUsers.fleet_read_only.password) + .set('kbn-xsrf', 'xxxx') + .expect(403); + }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/epm/get.ts b/x-pack/test/fleet_api_integration/apis/epm/get.ts index 014fe0808d255..13533a9a82af0 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/get.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/get.ts @@ -11,11 +11,13 @@ import path from 'path'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { setupFleetAndAgents } from '../agents/services'; +import { testUsers } from '../test_users'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); const testPkgKey = 'apache-0.1.4'; @@ -91,5 +93,12 @@ export default function (providerContext: FtrProviderContext) { it('returns a 400 for a package key without a proper semver version', async function () { await supertest.get('/api/fleet/epm/packages/endpoint-0.1.0.1.2.3').expect(400); }); + + it('allows user with only read permission to access', async () => { + await supertestWithoutAuth + .get(`/api/fleet/epm/packages/${testPkgKey}`) + .auth(testUsers.fleet_read_only.username, testUsers.fleet_read_only.password) + .expect(200); + }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts index 23feacbcee374..86928874f8a34 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -12,10 +12,12 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { setupFleetAndAgents } from '../agents/services'; +import { testUsers } from '../test_users'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); const dockerServers = getService('dockerServers'); const testPkgArchiveTgz = path.join( @@ -190,5 +192,16 @@ export default function (providerContext: FtrProviderContext) { '{"statusCode":400,"error":"Bad Request","message":"Name thisIsATypo and version 0.1.4 do not match top-level directory apache-0.1.4"}' ); }); + + it('should not allow users without all access', async () => { + const buf = fs.readFileSync(testPkgArchiveTgz); + await supertestWithoutAuth + .post(`/api/fleet/epm/packages`) + .auth(testUsers.fleet_read_only.username, testUsers.fleet_read_only.password) + .set('kbn-xsrf', 'xxxx') + .type('application/gzip') + .send(buf) + .expect(403); + }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/epm/list.ts b/x-pack/test/fleet_api_integration/apis/epm/list.ts index 931e494798220..56f3e6ca1f2fa 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/list.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/list.ts @@ -9,10 +9,12 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { setupFleetAndAgents } from '../agents/services'; +import { testUsers } from '../test_users'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); const esArchiver = getService('esArchiver'); // use function () {} and not () => {} here @@ -54,6 +56,13 @@ export default function (providerContext: FtrProviderContext) { expect(listResponse.response).to.eql(['endpoint']); }); + + it('allows user with only read permission to access', async () => { + await supertestWithoutAuth + .get('/api/fleet/epm/packages') + .auth(testUsers.fleet_read_only.username, testUsers.fleet_read_only.password) + .expect(200); + }); }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/index.js b/x-pack/test/fleet_api_integration/apis/index.js index 387433b787728..fd5ac05247fd2 100644 --- a/x-pack/test/fleet_api_integration/apis/index.js +++ b/x-pack/test/fleet_api_integration/apis/index.js @@ -5,10 +5,16 @@ * 2.0. */ -export default function ({ loadTestFile }) { +import { setupTestUsers } from './test_users'; + +export default function ({ loadTestFile, getService }) { describe('Fleet Endpoints', function () { + before(async () => { + await setupTestUsers(getService('security')); + }); + // EPM - loadTestFile(require.resolve('./epm/index')); + loadTestFile(require.resolve('./epm')); // Fleet setup loadTestFile(require.resolve('./fleet_setup')); diff --git a/x-pack/test/fleet_api_integration/apis/test_users.ts b/x-pack/test/fleet_api_integration/apis/test_users.ts new file mode 100644 index 0000000000000..a1df6d3a31b71 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/test_users.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SecurityService } from '../../../../test/common/services/security/security'; + +export const testUsers: { + [rollName: string]: { username: string; password: string; permissions?: any }; +} = { + fleet_no_access: { + permissions: { + feature: { + dashboards: ['read'], + }, + spaces: ['*'], + }, + username: 'fleet_no_access', + password: 'changeme', + }, + fleet_read_only: { + permissions: { + feature: { + fleet: ['read'], + }, + spaces: ['*'], + }, + username: 'fleet_read_only', + password: 'changeme', + }, + fleet_all: { + permissions: { + feature: { + fleet: ['all'], + }, + spaces: ['*'], + }, + username: 'fleet_all', + password: 'changeme', + }, +}; + +export const setupTestUsers = async (security: SecurityService) => { + for (const roleName in testUsers) { + if (testUsers.hasOwnProperty(roleName)) { + const user = testUsers[roleName]; + + if (user.permissions) { + await security.role.create(roleName, { + kibana: [user.permissions], + }); + } + + await security.user.create(user.username, { + password: user.password, + roles: [roleName], + full_name: user.username, + }); + } + } +}; From bfc790ae924bfa2d58c6d1fb54152b414846b38e Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Wed, 13 Oct 2021 12:46:11 -0500 Subject: [PATCH 07/35] fix codeowners file (#114783) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 17150e3c98cec..91a8f8c2d5998 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -63,7 +63,7 @@ /packages/kbn-interpreter/ @elastic/kibana-app-services /src/plugins/bfetch/ @elastic/kibana-app-services /src/plugins/data/ @elastic/kibana-app-services -/src/plugins/data-views/ @elastic/kibana-app-services +/src/plugins/data_views/ @elastic/kibana-app-services /src/plugins/embeddable/ @elastic/kibana-app-services /src/plugins/expressions/ @elastic/kibana-app-services /src/plugins/field_formats/ @elastic/kibana-app-services From eaf25d64e4b9dfa88371e0b54fc0ca0986359d13 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Oct 2021 19:48:01 +0200 Subject: [PATCH 08/35] [APM] Generate breakdown metrics (#114390) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- packages/elastic-apm-generator/src/index.ts | 1 + .../src/lib/base_span.ts | 31 +++- .../elastic-apm-generator/src/lib/entity.ts | 4 + .../src/lib/output/to_elasticsearch_output.ts | 4 +- .../elastic-apm-generator/src/lib/service.ts | 1 + .../elastic-apm-generator/src/lib/span.ts | 12 -- .../src/lib/transaction.ts | 28 ++-- .../src/lib/utils/aggregate.ts | 45 ++++++ .../src/lib/utils/create_picker.ts | 16 ++ .../src/lib/utils/get_breakdown_metrics.ts | 145 ++++++++++++++++++ .../lib/utils/get_span_destination_metrics.ts | 54 +++---- .../src/lib/utils/get_transaction_metrics.ts | 90 +++++------ .../src/scripts/examples/01_simple_trace.ts | 20 ++- .../test/scenarios/01_simple_trace.test.ts | 2 + .../scenarios/04_breakdown_metrics.test.ts | 105 +++++++++++++ .../01_simple_trace.test.ts.snap | 30 ++++ ...ervice_instances_transaction_statistics.ts | 3 + .../apm_api_integration/common/trace_data.ts | 8 +- .../instances_main_statistics.ts | 145 ++++++++++++++++++ 19 files changed, 621 insertions(+), 123 deletions(-) create mode 100644 packages/elastic-apm-generator/src/lib/utils/aggregate.ts create mode 100644 packages/elastic-apm-generator/src/lib/utils/create_picker.ts create mode 100644 packages/elastic-apm-generator/src/lib/utils/get_breakdown_metrics.ts create mode 100644 packages/elastic-apm-generator/src/test/scenarios/04_breakdown_metrics.test.ts diff --git a/packages/elastic-apm-generator/src/index.ts b/packages/elastic-apm-generator/src/index.ts index fd83ce483ad4f..7007e92012a66 100644 --- a/packages/elastic-apm-generator/src/index.ts +++ b/packages/elastic-apm-generator/src/index.ts @@ -12,3 +12,4 @@ export { getTransactionMetrics } from './lib/utils/get_transaction_metrics'; export { getSpanDestinationMetrics } from './lib/utils/get_span_destination_metrics'; export { getObserverDefaults } from './lib/defaults/get_observer_defaults'; export { toElasticsearchOutput } from './lib/output/to_elasticsearch_output'; +export { getBreakdownMetrics } from './lib/utils/get_breakdown_metrics'; diff --git a/packages/elastic-apm-generator/src/lib/base_span.ts b/packages/elastic-apm-generator/src/lib/base_span.ts index 24a51282687f4..6288c16d339b6 100644 --- a/packages/elastic-apm-generator/src/lib/base_span.ts +++ b/packages/elastic-apm-generator/src/lib/base_span.ts @@ -8,10 +8,12 @@ import { Fields } from './entity'; import { Serializable } from './serializable'; +import { Span } from './span'; +import { Transaction } from './transaction'; import { generateTraceId } from './utils/generate_id'; export class BaseSpan extends Serializable { - private _children: BaseSpan[] = []; + private readonly _children: BaseSpan[] = []; constructor(fields: Fields) { super({ @@ -22,20 +24,29 @@ export class BaseSpan extends Serializable { }); } - traceId(traceId: string) { - this.fields['trace.id'] = traceId; + parent(span: BaseSpan) { + this.fields['trace.id'] = span.fields['trace.id']; + this.fields['parent.id'] = span.isSpan() + ? span.fields['span.id'] + : span.fields['transaction.id']; + + if (this.isSpan()) { + this.fields['transaction.id'] = span.fields['transaction.id']; + } this._children.forEach((child) => { - child.fields['trace.id'] = traceId; + child.parent(this); }); + return this; } children(...children: BaseSpan[]) { - this._children.push(...children); children.forEach((child) => { - child.traceId(this.fields['trace.id']!); + child.parent(this); }); + this._children.push(...children); + return this; } @@ -52,4 +63,12 @@ export class BaseSpan extends Serializable { serialize(): Fields[] { return [this.fields, ...this._children.flatMap((child) => child.serialize())]; } + + isSpan(): this is Span { + return this.fields['processor.event'] === 'span'; + } + + isTransaction(): this is Transaction { + return this.fields['processor.event'] === 'transaction'; + } } diff --git a/packages/elastic-apm-generator/src/lib/entity.ts b/packages/elastic-apm-generator/src/lib/entity.ts index e0a048c876213..2a4beee652cf7 100644 --- a/packages/elastic-apm-generator/src/lib/entity.ts +++ b/packages/elastic-apm-generator/src/lib/entity.ts @@ -10,9 +10,11 @@ export type Fields = Partial<{ '@timestamp': number; 'agent.name': string; 'agent.version': string; + 'container.id': string; 'ecs.version': string; 'event.outcome': string; 'event.ingested': number; + 'host.name': string; 'metricset.name': string; 'observer.version': string; 'observer.version_major': number; @@ -42,6 +44,8 @@ export type Fields = Partial<{ 'span.destination.service.type': string; 'span.destination.service.response_time.sum.us': number; 'span.destination.service.response_time.count': number; + 'span.self_time.count': number; + 'span.self_time.sum.us': number; }>; export class Entity { diff --git a/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts b/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts index ded94f9ad2276..b4cae1b41b9a6 100644 --- a/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts +++ b/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts @@ -14,10 +14,12 @@ export function toElasticsearchOutput(events: Fields[], versionOverride?: string return events.map((event) => { const values = { ...event, + ...getObserverDefaults(), '@timestamp': new Date(event['@timestamp']!).toISOString(), 'timestamp.us': event['@timestamp']! * 1000, 'ecs.version': '1.4', - ...getObserverDefaults(), + 'service.node.name': + event['service.node.name'] || event['container.id'] || event['host.name'], }; const document = {}; diff --git a/packages/elastic-apm-generator/src/lib/service.ts b/packages/elastic-apm-generator/src/lib/service.ts index 8ddbd827e842e..859afa18aab03 100644 --- a/packages/elastic-apm-generator/src/lib/service.ts +++ b/packages/elastic-apm-generator/src/lib/service.ts @@ -14,6 +14,7 @@ export class Service extends Entity { return new Instance({ ...this.fields, ['service.node.name']: instanceName, + 'container.id': instanceName, }); } } diff --git a/packages/elastic-apm-generator/src/lib/span.ts b/packages/elastic-apm-generator/src/lib/span.ts index da9ba9cdff722..36f7f44816d01 100644 --- a/packages/elastic-apm-generator/src/lib/span.ts +++ b/packages/elastic-apm-generator/src/lib/span.ts @@ -19,18 +19,6 @@ export class Span extends BaseSpan { }); } - children(...children: BaseSpan[]) { - super.children(...children); - - children.forEach((child) => - child.defaults({ - 'parent.id': this.fields['span.id'], - }) - ); - - return this; - } - duration(duration: number) { this.fields['span.duration.us'] = duration * 1000; return this; diff --git a/packages/elastic-apm-generator/src/lib/transaction.ts b/packages/elastic-apm-generator/src/lib/transaction.ts index 14ed6ac1ea85e..f615f46710996 100644 --- a/packages/elastic-apm-generator/src/lib/transaction.ts +++ b/packages/elastic-apm-generator/src/lib/transaction.ts @@ -11,6 +11,8 @@ import { Fields } from './entity'; import { generateEventId } from './utils/generate_id'; export class Transaction extends BaseSpan { + private _sampled: boolean = true; + constructor(fields: Fields) { super({ ...fields, @@ -19,19 +21,25 @@ export class Transaction extends BaseSpan { 'transaction.sampled': true, }); } - children(...children: BaseSpan[]) { - super.children(...children); - children.forEach((child) => - child.defaults({ - 'transaction.id': this.fields['transaction.id'], - 'parent.id': this.fields['transaction.id'], - }) - ); - return this; - } duration(duration: number) { this.fields['transaction.duration.us'] = duration * 1000; return this; } + + sample(sampled: boolean = true) { + this._sampled = sampled; + return this; + } + + serialize() { + const [transaction, ...spans] = super.serialize(); + + const events = [transaction]; + if (this._sampled) { + events.push(...spans); + } + + return events; + } } diff --git a/packages/elastic-apm-generator/src/lib/utils/aggregate.ts b/packages/elastic-apm-generator/src/lib/utils/aggregate.ts new file mode 100644 index 0000000000000..81b72f6fa01e9 --- /dev/null +++ b/packages/elastic-apm-generator/src/lib/utils/aggregate.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import moment from 'moment'; +import { pickBy } from 'lodash'; +import objectHash from 'object-hash'; +import { Fields } from '../entity'; +import { createPicker } from './create_picker'; + +export function aggregate(events: Fields[], fields: string[]) { + const picker = createPicker(fields); + + const metricsets = new Map(); + + function getMetricsetKey(span: Fields) { + const timestamp = moment(span['@timestamp']).valueOf(); + return { + '@timestamp': timestamp - (timestamp % (60 * 1000)), + ...pickBy(span, picker), + }; + } + + for (const event of events) { + const key = getMetricsetKey(event); + const id = objectHash(key); + + let metricset = metricsets.get(id); + + if (!metricset) { + metricset = { + key: { ...key, 'processor.event': 'metric', 'processor.name': 'metric' }, + events: [], + }; + metricsets.set(id, metricset); + } + + metricset.events.push(event); + } + + return Array.from(metricsets.values()); +} diff --git a/packages/elastic-apm-generator/src/lib/utils/create_picker.ts b/packages/elastic-apm-generator/src/lib/utils/create_picker.ts new file mode 100644 index 0000000000000..7fce23b6fc966 --- /dev/null +++ b/packages/elastic-apm-generator/src/lib/utils/create_picker.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +export function createPicker(fields: string[]) { + const wildcards = fields + .filter((field) => field.endsWith('.*')) + .map((field) => field.replace('*', '')); + + return (value: unknown, key: string) => { + return fields.includes(key) || wildcards.some((field) => key.startsWith(field)); + }; +} diff --git a/packages/elastic-apm-generator/src/lib/utils/get_breakdown_metrics.ts b/packages/elastic-apm-generator/src/lib/utils/get_breakdown_metrics.ts new file mode 100644 index 0000000000000..8eae0941c6bdd --- /dev/null +++ b/packages/elastic-apm-generator/src/lib/utils/get_breakdown_metrics.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import objectHash from 'object-hash'; +import { groupBy, pickBy } from 'lodash'; +import { Fields } from '../entity'; +import { createPicker } from './create_picker'; + +const instanceFields = [ + 'container.*', + 'kubernetes.*', + 'agent.*', + 'process.*', + 'cloud.*', + 'service.*', + 'host.*', +]; + +const instancePicker = createPicker(instanceFields); + +const metricsetPicker = createPicker([ + 'transaction.type', + 'transaction.name', + 'span.type', + 'span.subtype', +]); + +export function getBreakdownMetrics(events: Fields[]) { + const txWithSpans = groupBy( + events.filter( + (event) => event['processor.event'] === 'span' || event['processor.event'] === 'transaction' + ), + (event) => event['transaction.id'] + ); + + const metricsets: Map = new Map(); + + Object.keys(txWithSpans).forEach((transactionId) => { + const txEvents = txWithSpans[transactionId]; + const transaction = txEvents.find((event) => event['processor.event'] === 'transaction')!; + + const eventsById: Record = {}; + const activityByParentId: Record> = {}; + for (const event of txEvents) { + const id = + event['processor.event'] === 'transaction' ? event['transaction.id'] : event['span.id']; + eventsById[id!] = event; + + const parentId = event['parent.id']; + + if (!parentId) { + continue; + } + + if (!activityByParentId[parentId]) { + activityByParentId[parentId] = []; + } + + const from = event['@timestamp']! * 1000; + const to = + from + + (event['processor.event'] === 'transaction' + ? event['transaction.duration.us']! + : event['span.duration.us']!); + + activityByParentId[parentId].push({ from, to }); + } + + // eslint-disable-next-line guard-for-in + for (const id in eventsById) { + const event = eventsById[id]; + const activities = activityByParentId[id] || []; + + const timeStart = event['@timestamp']! * 1000; + + let selfTime = 0; + let lastMeasurement = timeStart; + const changeTimestamps = [ + ...new Set([ + timeStart, + ...activities.flatMap((activity) => [activity.from, activity.to]), + timeStart + + (event['processor.event'] === 'transaction' + ? event['transaction.duration.us']! + : event['span.duration.us']!), + ]), + ]; + + for (const timestamp of changeTimestamps) { + const hasActiveChildren = activities.some( + (activity) => activity.from < timestamp && activity.to >= timestamp + ); + + if (!hasActiveChildren) { + selfTime += timestamp - lastMeasurement; + } + + lastMeasurement = timestamp; + } + + const key = { + '@timestamp': event['@timestamp']! - (event['@timestamp']! % (30 * 1000)), + 'transaction.type': transaction['transaction.type'], + 'transaction.name': transaction['transaction.name'], + ...pickBy(event, metricsetPicker), + }; + + const instance = pickBy(event, instancePicker); + + const metricsetId = objectHash(key); + + let metricset = metricsets.get(metricsetId); + + if (!metricset) { + metricset = { + ...key, + ...instance, + 'processor.event': 'metric', + 'processor.name': 'metric', + 'metricset.name': `span_breakdown`, + 'span.self_time.count': 0, + 'span.self_time.sum.us': 0, + }; + + if (event['processor.event'] === 'transaction') { + metricset['span.type'] = 'app'; + } else { + metricset['span.type'] = event['span.type']; + metricset['span.subtype'] = event['span.subtype']; + } + + metricsets.set(metricsetId, metricset); + } + + metricset['span.self_time.count']!++; + metricset['span.self_time.sum.us']! += selfTime; + } + }); + + return Array.from(metricsets.values()); +} diff --git a/packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts b/packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts index 3740ad685735e..decf2f71a9be4 100644 --- a/packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts +++ b/packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts @@ -6,46 +6,34 @@ * Side Public License, v 1. */ -import { pick } from 'lodash'; -import moment from 'moment'; -import objectHash from 'object-hash'; import { Fields } from '../entity'; +import { aggregate } from './aggregate'; export function getSpanDestinationMetrics(events: Fields[]) { const exitSpans = events.filter((event) => !!event['span.destination.service.resource']); - const metricsets = new Map(); + const metricsets = aggregate(exitSpans, [ + 'event.outcome', + 'agent.name', + 'service.environment', + 'service.name', + 'span.destination.service.resource', + ]); - function getSpanBucketKey(span: Fields) { - return { - '@timestamp': moment(span['@timestamp']).startOf('minute').valueOf(), - ...pick(span, [ - 'event.outcome', - 'agent.name', - 'service.environment', - 'service.name', - 'span.destination.service.resource', - ]), - }; - } - - for (const span of exitSpans) { - const key = getSpanBucketKey(span); - const id = objectHash(key); + return metricsets.map((metricset) => { + let count = 0; + let sum = 0; - let metricset = metricsets.get(id); - if (!metricset) { - metricset = { - ['processor.event']: 'metric', - ...key, - 'span.destination.service.response_time.sum.us': 0, - 'span.destination.service.response_time.count': 0, - }; - metricsets.set(id, metricset); + for (const event of metricset.events) { + count++; + sum += event['span.duration.us']!; } - metricset['span.destination.service.response_time.count']! += 1; - metricset['span.destination.service.response_time.sum.us']! += span['span.duration.us']!; - } - return [...Array.from(metricsets.values())]; + return { + ...metricset.key, + ['metricset.name']: 'span_destination', + 'span.destination.service.response_time.sum.us': sum, + 'span.destination.service.response_time.count': count, + }; + }); } diff --git a/packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts b/packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts index 62ecb9e20006f..4d46461c6dcc9 100644 --- a/packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts +++ b/packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts @@ -6,10 +6,9 @@ * Side Public License, v 1. */ -import { pick, sortBy } from 'lodash'; -import moment from 'moment'; -import objectHash from 'object-hash'; +import { sortBy } from 'lodash'; import { Fields } from '../entity'; +import { aggregate } from './aggregate'; function sortAndCompressHistogram(histogram?: { values: number[]; counts: number[] }) { return sortBy(histogram?.values).reduce( @@ -30,60 +29,45 @@ function sortAndCompressHistogram(histogram?: { values: number[]; counts: number } export function getTransactionMetrics(events: Fields[]) { - const transactions = events.filter((event) => event['processor.event'] === 'transaction'); + const transactions = events + .filter((event) => event['processor.event'] === 'transaction') + .map((transaction) => { + return { + ...transaction, + ['trace.root']: transaction['parent.id'] === undefined, + }; + }); - const metricsets = new Map(); + const metricsets = aggregate(transactions, [ + 'trace.root', + 'transaction.name', + 'transaction.type', + 'event.outcome', + 'transaction.result', + 'agent.name', + 'service.environment', + 'service.name', + 'service.version', + 'host.name', + 'container.id', + 'kubernetes.pod.name', + ]); - function getTransactionBucketKey(transaction: Fields) { - return { - '@timestamp': moment(transaction['@timestamp']).startOf('minute').valueOf(), - 'trace.root': transaction['parent.id'] === undefined, - ...pick(transaction, [ - 'transaction.name', - 'transaction.type', - 'event.outcome', - 'transaction.result', - 'agent.name', - 'service.environment', - 'service.name', - 'service.version', - 'host.name', - 'container.id', - 'kubernetes.pod.name', - ]), + return metricsets.map((metricset) => { + const histogram = { + values: [] as number[], + counts: [] as number[], }; - } - for (const transaction of transactions) { - const key = getTransactionBucketKey(transaction); - const id = objectHash(key); - let metricset = metricsets.get(id); - if (!metricset) { - metricset = { - ...key, - ['processor.event']: 'metric', - 'transaction.duration.histogram': { - values: [], - counts: [], - }, - }; - metricsets.set(id, metricset); + for (const transaction of metricset.events) { + histogram.counts.push(1); + histogram.values.push(Number(transaction['transaction.duration.us'])); } - metricset['transaction.duration.histogram']?.counts.push(1); - metricset['transaction.duration.histogram']?.values.push( - Number(transaction['transaction.duration.us']) - ); - } - return [ - ...Array.from(metricsets.values()).map((metricset) => { - return { - ...metricset, - ['transaction.duration.histogram']: sortAndCompressHistogram( - metricset['transaction.duration.histogram'] - ), - _doc_count: metricset['transaction.duration.histogram']!.values.length, - }; - }), - ]; + return { + ...metricset.key, + 'transaction.duration.histogram': sortAndCompressHistogram(histogram), + _doc_count: metricset.events.length, + }; + }); } diff --git a/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts b/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts index eef3e6cc40560..7aae2986919c8 100644 --- a/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts +++ b/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts @@ -7,17 +7,18 @@ */ import { service, timerange, getTransactionMetrics, getSpanDestinationMetrics } from '../..'; +import { getBreakdownMetrics } from '../../lib/utils/get_breakdown_metrics'; export function simpleTrace(from: number, to: number) { const instance = service('opbeans-go', 'production', 'go').instance('instance'); const range = timerange(from, to); - const transactionName = '100rpm (75% success) failed 1000ms'; + const transactionName = '100rpm (80% success) failed 1000ms'; const successfulTraceEvents = range - .interval('1m') - .rate(75) + .interval('30s') + .rate(40) .flatMap((timestamp) => instance .transaction(transactionName) @@ -31,14 +32,14 @@ export function simpleTrace(from: number, to: number) { .success() .destination('elasticsearch') .timestamp(timestamp), - instance.span('custom_operation', 'app').duration(50).success().timestamp(timestamp) + instance.span('custom_operation', 'custom').duration(100).success().timestamp(timestamp) ) .serialize() ); const failedTraceEvents = range - .interval('1m') - .rate(25) + .interval('30s') + .rate(10) .flatMap((timestamp) => instance .transaction(transactionName) @@ -50,5 +51,10 @@ export function simpleTrace(from: number, to: number) { const events = successfulTraceEvents.concat(failedTraceEvents); - return events.concat(getTransactionMetrics(events)).concat(getSpanDestinationMetrics(events)); + return [ + ...events, + ...getTransactionMetrics(events), + ...getSpanDestinationMetrics(events), + ...getBreakdownMetrics(events), + ]; } diff --git a/packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts b/packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts index 6bae70507dcbe..733093ce0a71c 100644 --- a/packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts +++ b/packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts @@ -68,6 +68,7 @@ describe('simple trace', () => { expect(transaction).toEqual({ '@timestamp': 1609459200000, 'agent.name': 'java', + 'container.id': 'instance-1', 'event.outcome': 'success', 'processor.event': 'transaction', 'processor.name': 'transaction', @@ -89,6 +90,7 @@ describe('simple trace', () => { expect(span).toEqual({ '@timestamp': 1609459200050, 'agent.name': 'java', + 'container.id': 'instance-1', 'event.outcome': 'success', 'parent.id': 'e7433020f2745625', 'processor.event': 'span', diff --git a/packages/elastic-apm-generator/src/test/scenarios/04_breakdown_metrics.test.ts b/packages/elastic-apm-generator/src/test/scenarios/04_breakdown_metrics.test.ts new file mode 100644 index 0000000000000..aeb944f35faf6 --- /dev/null +++ b/packages/elastic-apm-generator/src/test/scenarios/04_breakdown_metrics.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { sumBy } from 'lodash'; +import { Fields } from '../../lib/entity'; +import { service } from '../../lib/service'; +import { timerange } from '../../lib/timerange'; +import { getBreakdownMetrics } from '../../lib/utils/get_breakdown_metrics'; + +describe('breakdown metrics', () => { + let events: Fields[]; + + const LIST_RATE = 2; + const LIST_SPANS = 2; + const ID_RATE = 4; + const ID_SPANS = 2; + const INTERVALS = 6; + + beforeEach(() => { + const javaService = service('opbeans-java', 'production', 'java'); + const javaInstance = javaService.instance('instance-1'); + + const start = new Date('2021-01-01T00:00:00.000Z').getTime(); + + const range = timerange(start, start + INTERVALS * 30 * 1000 - 1); + + events = getBreakdownMetrics([ + ...range + .interval('30s') + .rate(LIST_RATE) + .flatMap((timestamp) => + javaInstance + .transaction('GET /api/product/list') + .timestamp(timestamp) + .duration(1000) + .children( + javaInstance + .span('GET apm-*/_search', 'db', 'elasticsearch') + .timestamp(timestamp + 150) + .duration(500), + javaInstance.span('GET foo', 'db', 'redis').timestamp(timestamp).duration(100) + ) + .serialize() + ), + ...range + .interval('30s') + .rate(ID_RATE) + .flatMap((timestamp) => + javaInstance + .transaction('GET /api/product/:id') + .timestamp(timestamp) + .duration(1000) + .children( + javaInstance + .span('GET apm-*/_search', 'db', 'elasticsearch') + .duration(500) + .timestamp(timestamp + 100) + .children( + javaInstance + .span('bar', 'external', 'http') + .timestamp(timestamp + 200) + .duration(100) + ) + ) + .serialize() + ), + ]).filter((event) => event['processor.event'] === 'metric'); + }); + + it('generates the right amount of breakdown metrics', () => { + expect(events.length).toBe(INTERVALS * (LIST_SPANS + 1 + ID_SPANS + 1)); + }); + + it('calculates breakdown metrics for the right amount of transactions and spans', () => { + expect(sumBy(events, (event) => event['span.self_time.count']!)).toBe( + INTERVALS * LIST_RATE * (LIST_SPANS + 1) + INTERVALS * ID_RATE * (ID_SPANS + 1) + ); + }); + + it('generates app metricsets for transaction self time', () => { + expect(events.some((event) => event['span.type'] === 'app' && !event['span.subtype'])).toBe( + true + ); + }); + + it('generates the right statistic', () => { + const elasticsearchSets = events.filter((event) => event['span.subtype'] === 'elasticsearch'); + + const expectedCountFromListTransaction = INTERVALS * LIST_RATE; + + const expectedCountFromIdTransaction = INTERVALS * ID_RATE; + + const expectedCount = expectedCountFromIdTransaction + expectedCountFromListTransaction; + + expect(sumBy(elasticsearchSets, (set) => set['span.self_time.count']!)).toBe(expectedCount); + + expect(sumBy(elasticsearchSets, (set) => set['span.self_time.sum.us']!)).toBe( + expectedCountFromListTransaction * 500 * 1000 + expectedCountFromIdTransaction * 400 * 1000 + ); + }); +}); diff --git a/packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap b/packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap index 6eec0ce38ba30..00a55cb87b125 100644 --- a/packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap +++ b/packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap @@ -5,6 +5,7 @@ Array [ Object { "@timestamp": 1609459200000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -21,6 +22,7 @@ Array [ Object { "@timestamp": 1609459200050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "36c16f18e75058f8", "processor.event": "span", @@ -39,6 +41,7 @@ Array [ Object { "@timestamp": 1609459260000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -55,6 +58,7 @@ Array [ Object { "@timestamp": 1609459260050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "65ce74106eb050be", "processor.event": "span", @@ -73,6 +77,7 @@ Array [ Object { "@timestamp": 1609459320000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -89,6 +94,7 @@ Array [ Object { "@timestamp": 1609459320050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "91fa709d90625fff", "processor.event": "span", @@ -107,6 +113,7 @@ Array [ Object { "@timestamp": 1609459380000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -123,6 +130,7 @@ Array [ Object { "@timestamp": 1609459380050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "6c500d1d19835e68", "processor.event": "span", @@ -141,6 +149,7 @@ Array [ Object { "@timestamp": 1609459440000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -157,6 +166,7 @@ Array [ Object { "@timestamp": 1609459440050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "1b3246cc83595869", "processor.event": "span", @@ -175,6 +185,7 @@ Array [ Object { "@timestamp": 1609459500000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -191,6 +202,7 @@ Array [ Object { "@timestamp": 1609459500050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "12b49e3c83fe58d5", "processor.event": "span", @@ -209,6 +221,7 @@ Array [ Object { "@timestamp": 1609459560000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -225,6 +238,7 @@ Array [ Object { "@timestamp": 1609459560050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "d9272009dd4354a1", "processor.event": "span", @@ -243,6 +257,7 @@ Array [ Object { "@timestamp": 1609459620000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -259,6 +274,7 @@ Array [ Object { "@timestamp": 1609459620050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "bc52ca08063c505b", "processor.event": "span", @@ -277,6 +293,7 @@ Array [ Object { "@timestamp": 1609459680000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -293,6 +310,7 @@ Array [ Object { "@timestamp": 1609459680050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "186858dd88b75d59", "processor.event": "span", @@ -311,6 +329,7 @@ Array [ Object { "@timestamp": 1609459740000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -327,6 +346,7 @@ Array [ Object { "@timestamp": 1609459740050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "0d5f44d48189546c", "processor.event": "span", @@ -345,6 +365,7 @@ Array [ Object { "@timestamp": 1609459800000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -361,6 +382,7 @@ Array [ Object { "@timestamp": 1609459800050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "7483e0606e435c83", "processor.event": "span", @@ -379,6 +401,7 @@ Array [ Object { "@timestamp": 1609459860000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -395,6 +418,7 @@ Array [ Object { "@timestamp": 1609459860050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "f142c4cbc7f3568e", "processor.event": "span", @@ -413,6 +437,7 @@ Array [ Object { "@timestamp": 1609459920000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -429,6 +454,7 @@ Array [ Object { "@timestamp": 1609459920050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "2e3a47fa2d905519", "processor.event": "span", @@ -447,6 +473,7 @@ Array [ Object { "@timestamp": 1609459980000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -463,6 +490,7 @@ Array [ Object { "@timestamp": 1609459980050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "de5eaa1e47dc56b1", "processor.event": "span", @@ -481,6 +509,7 @@ Array [ Object { "@timestamp": 1609460040000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -497,6 +526,7 @@ Array [ Object { "@timestamp": 1609460040050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "af7eac7ae61e576a", "processor.event": "span", diff --git a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts index 089282d6f1c34..ec76e0d35e5c0 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts @@ -109,6 +109,9 @@ export async function getServiceInstancesTransactionStatistics< filter: [ { term: { [SERVICE_NAME]: serviceName } }, { term: { [TRANSACTION_TYPE]: transactionType } }, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), ...rangeQuery(start, end), ...environmentQuery(environment), ...kqlQuery(kuery), diff --git a/x-pack/test/apm_api_integration/common/trace_data.ts b/x-pack/test/apm_api_integration/common/trace_data.ts index 9c96d3fa1e0b0..84bbb4beea4f4 100644 --- a/x-pack/test/apm_api_integration/common/trace_data.ts +++ b/x-pack/test/apm_api_integration/common/trace_data.ts @@ -6,6 +6,7 @@ */ import { + getBreakdownMetrics, getSpanDestinationMetrics, getTransactionMetrics, toElasticsearchOutput, @@ -20,7 +21,12 @@ export async function traceData(context: InheritedFtrProviderContext) { return { index: (events: any[]) => { const esEvents = toElasticsearchOutput( - events.concat(getTransactionMetrics(events)).concat(getSpanDestinationMetrics(events)), + [ + ...events, + ...getTransactionMetrics(events), + ...getSpanDestinationMetrics(events), + ...getBreakdownMetrics(events), + ], '7.14.0' ); diff --git a/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.ts b/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.ts index 2d165f4ceb902..cdf62053a821b 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.ts +++ b/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.ts @@ -8,6 +8,7 @@ import expect from '@kbn/expect'; import { pick, sortBy } from 'lodash'; import moment from 'moment'; +import { service, timerange } from '@elastic/apm-generator'; import { APIReturnType } from '../../../../plugins/apm/public/services/rest/createCallApmApi'; import { isFiniteNumber } from '../../../../plugins/apm/common/utils/is_finite_number'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -15,9 +16,12 @@ import archives from '../../common/fixtures/es_archiver/archives_metadata'; import { registry } from '../../common/registry'; import { LatencyAggregationType } from '../../../../plugins/apm/common/latency_aggregation_types'; +import { ENVIRONMENT_ALL } from '../../../../plugins/apm/common/environment_filter_values'; +import { SERVICE_NODE_NAME_MISSING } from '../../../../plugins/apm/common/service_nodes'; export default function ApiTest({ getService }: FtrProviderContext) { const apmApiClient = getService('apmApiClient'); + const traceData = getService('traceData'); const archiveName = 'apm_8.0.0'; const { start, end } = archives[archiveName]; @@ -278,4 +282,145 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); } ); + + registry.when( + 'Service overview instances main statistics when data is generated', + { config: 'basic', archives: ['apm_8.0.0_empty'] }, + () => { + describe('for two go instances and one java instance', () => { + const GO_A_INSTANCE_RATE_SUCCESS = 10; + const GO_A_INSTANCE_RATE_FAILURE = 5; + const GO_B_INSTANCE_RATE_SUCCESS = 15; + + const JAVA_INSTANCE_RATE = 20; + + const rangeStart = new Date('2021-01-01T12:00:00.000Z').getTime(); + const rangeEnd = new Date('2021-01-01T12:15:00.000Z').getTime() - 1; + + before(async () => { + const goService = service('opbeans-go', 'production', 'go'); + const javaService = service('opbeans-java', 'production', 'java'); + + const goInstanceA = goService.instance('go-instance-a'); + const goInstanceB = goService.instance('go-instance-b'); + const javaInstance = javaService.instance('java-instance'); + + const interval = timerange(rangeStart, rangeEnd).interval('1m'); + + // include exit spans to generate span_destination metrics + // that should not be included + function withSpans(timestamp: number) { + return new Array(3).fill(undefined).map(() => + goInstanceA + .span('GET apm-*/_search', 'db', 'elasticsearch') + .timestamp(timestamp + 100) + .duration(300) + .destination('elasticsearch') + .success() + ); + } + + return traceData.index([ + ...interval.rate(GO_A_INSTANCE_RATE_SUCCESS).flatMap((timestamp) => + goInstanceA + .transaction('GET /api/product/list') + .success() + .duration(500) + .timestamp(timestamp) + .children(...withSpans(timestamp)) + .serialize() + ), + ...interval.rate(GO_A_INSTANCE_RATE_FAILURE).flatMap((timestamp) => + goInstanceA + .transaction('GET /api/product/list') + .failure() + .duration(500) + .timestamp(timestamp) + .children(...withSpans(timestamp)) + .serialize() + ), + ...interval.rate(GO_B_INSTANCE_RATE_SUCCESS).flatMap((timestamp) => + goInstanceB + .transaction('GET /api/product/list') + .success() + .duration(500) + .timestamp(timestamp) + .children(...withSpans(timestamp)) + .serialize() + ), + ...interval.rate(JAVA_INSTANCE_RATE).flatMap((timestamp) => + javaInstance + .transaction('GET /api/product/list') + .success() + .duration(500) + .timestamp(timestamp) + .children(...withSpans(timestamp)) + .serialize() + ), + ]); + }); + + after(async () => { + return traceData.clean(); + }); + + describe('for the go service', () => { + let body: APIReturnType<'GET /internal/apm/services/{serviceName}/service_overview_instances/main_statistics'>; + + before(async () => { + body = ( + await apmApiClient.readUser({ + endpoint: + 'GET /internal/apm/services/{serviceName}/service_overview_instances/main_statistics', + params: { + path: { + serviceName: 'opbeans-go', + }, + query: { + start: new Date(rangeStart).toISOString(), + end: new Date(rangeEnd + 1).toISOString(), + environment: ENVIRONMENT_ALL.value, + kuery: '', + latencyAggregationType: LatencyAggregationType.avg, + transactionType: 'request', + }, + }, + }) + ).body; + }); + + it('returns statistics for the go instances', () => { + const goAStats = body.currentPeriod.find( + (stat) => stat.serviceNodeName === 'go-instance-a' + ); + const goBStats = body.currentPeriod.find( + (stat) => stat.serviceNodeName === 'go-instance-b' + ); + + expect(goAStats?.throughput).to.eql( + GO_A_INSTANCE_RATE_SUCCESS + GO_A_INSTANCE_RATE_FAILURE + ); + + expect(goBStats?.throughput).to.eql(GO_B_INSTANCE_RATE_SUCCESS); + }); + + it('does not return data for the java service', () => { + const javaStats = body.currentPeriod.find( + (stat) => stat.serviceNodeName === 'java-instance' + ); + + expect(javaStats).to.be(undefined); + }); + + it('does not return data for missing service node name', () => { + const missingNameStats = body.currentPeriod.find( + (stat) => stat.serviceNodeName === SERVICE_NODE_NAME_MISSING + ); + + expect(missingNameStats).to.be(undefined); + }); + }); + }); + } + ); } From 6d5354a99df279221166648fb6c5fa3381201d81 Mon Sep 17 00:00:00 2001 From: Clint Andrew Hall Date: Wed, 13 Oct 2021 12:50:20 -0500 Subject: [PATCH 09/35] [fleet][integrations] Provide Deployment Details on Cloud (#114287) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...-plugin-core-public.doclinksstart.links.md | 1 + .../public/doc_links/doc_links_service.ts | 2 + src/core/public/public.api.md | 1 + x-pack/plugins/fleet/kibana.json | 2 +- .../public/applications/integrations/app.tsx | 77 ++++++----- .../header/deployment_details.component.tsx | 120 ++++++++++++++++++ .../header/deployment_details.stories.tsx | 56 ++++++++ .../components/header/deployment_details.tsx | 51 ++++++++ .../integrations/components/header/header.tsx | 32 +++++ .../components/header/header_portal.tsx | 35 +++++ .../integrations/components/header/index.ts | 8 ++ .../applications/integrations/index.tsx | 6 +- .../public/mock/create_test_renderer.tsx | 4 + .../fleet/public/mock/plugin_dependencies.ts | 2 + x-pack/plugins/fleet/public/plugin.ts | 5 + .../plugins/fleet/storybook/context/cloud.ts | 23 ++++ .../fleet/storybook/context/doc_links.ts | 1 + .../plugins/fleet/storybook/context/index.tsx | 27 +++- .../plugins/fleet/storybook/context/share.ts | 22 ++++ .../plugins/fleet/storybook/context/stubs.tsx | 4 +- x-pack/plugins/fleet/storybook/decorator.tsx | 2 +- 21 files changed, 436 insertions(+), 45 deletions(-) create mode 100644 x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.component.tsx create mode 100644 x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.stories.tsx create mode 100644 x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.tsx create mode 100644 x-pack/plugins/fleet/public/applications/integrations/components/header/header.tsx create mode 100644 x-pack/plugins/fleet/public/applications/integrations/components/header/header_portal.tsx create mode 100644 x-pack/plugins/fleet/public/applications/integrations/components/header/index.ts create mode 100644 x-pack/plugins/fleet/storybook/context/cloud.ts create mode 100644 x-pack/plugins/fleet/storybook/context/share.ts diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md index 055cf213dea90..e79bc7a0db026 100644 --- a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md +++ b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md @@ -240,6 +240,7 @@ readonly links: { upgradeElasticAgent: string; upgradeElasticAgent712lower: string; learnMoreBlog: string; + apiKeysLearnMore: string; }>; readonly ecs: { readonly guide: string; diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts index 6039a0766a1a1..ac0aac3466f5f 100644 --- a/src/core/public/doc_links/doc_links_service.ts +++ b/src/core/public/doc_links/doc_links_service.ts @@ -482,6 +482,7 @@ export class DocLinksService { upgradeElasticAgent: `${FLEET_DOCS}upgrade-elastic-agent.html`, upgradeElasticAgent712lower: `${FLEET_DOCS}upgrade-elastic-agent.html#upgrade-7.12-lower`, learnMoreBlog: `${ELASTIC_WEBSITE_URL}blog/elastic-agent-and-fleet-make-it-easier-to-integrate-your-systems-with-elastic`, + apiKeysLearnMore: `${KIBANA_DOCS}api-keys.html`, }, ecs: { guide: `${ELASTIC_WEBSITE_URL}guide/en/ecs/current/index.html`, @@ -741,6 +742,7 @@ export interface DocLinksStart { upgradeElasticAgent: string; upgradeElasticAgent712lower: string; learnMoreBlog: string; + apiKeysLearnMore: string; }>; readonly ecs: { readonly guide: string; diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 7b0ec39d4a4d9..50c9bcf925969 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -709,6 +709,7 @@ export interface DocLinksStart { upgradeElasticAgent: string; upgradeElasticAgent712lower: string; learnMoreBlog: string; + apiKeysLearnMore: string; }>; readonly ecs: { readonly guide: string; diff --git a/x-pack/plugins/fleet/kibana.json b/x-pack/plugins/fleet/kibana.json index c4782156b1982..9de538ee91b8c 100644 --- a/x-pack/plugins/fleet/kibana.json +++ b/x-pack/plugins/fleet/kibana.json @@ -8,7 +8,7 @@ "server": true, "ui": true, "configPath": ["xpack", "fleet"], - "requiredPlugins": ["licensing", "data", "encryptedSavedObjects", "navigation", "customIntegrations"], + "requiredPlugins": ["licensing", "data", "encryptedSavedObjects", "navigation", "customIntegrations", "share"], "optionalPlugins": ["security", "features", "cloud", "usageCollection", "home", "globalSearch"], "extraPublicDirs": ["common"], "requiredBundles": ["kibanaReact", "esUiShared", "home", "infra", "kibanaUtils"] diff --git a/x-pack/plugins/fleet/public/applications/integrations/app.tsx b/x-pack/plugins/fleet/public/applications/integrations/app.tsx index 771b17ae8c3ee..c2f6f53627e38 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/app.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/app.tsx @@ -40,6 +40,7 @@ import { EPMApp } from './sections/epm'; import { DefaultLayout } from './layouts'; import { PackageInstallProvider } from './hooks'; import { useBreadcrumbs, UIExtensionsContext } from './hooks'; +import { IntegrationsHeader } from './components/header'; const ErrorLayout = ({ children }: { children: JSX.Element }) => ( @@ -127,41 +128,53 @@ export const IntegrationsAppContext: React.FC<{ history: AppMountParameters['history']; kibanaVersion: string; extensions: UIExtensionsStorage; + setHeaderActionMenu: AppMountParameters['setHeaderActionMenu']; /** For testing purposes only */ routerHistory?: History; // TODO remove -}> = memo(({ children, startServices, config, history, kibanaVersion, extensions }) => { - const isDarkMode = useObservable(startServices.uiSettings.get$('theme:darkMode')); +}> = memo( + ({ + children, + startServices, + config, + history, + kibanaVersion, + extensions, + setHeaderActionMenu, + }) => { + const isDarkMode = useObservable(startServices.uiSettings.get$('theme:darkMode')); - return ( - - - - - - - - - - - - - - {children} - - - - - - - - - - - - - - ); -}); + return ( + + + + + + + + + + + + + + + {children} + + + + + + + + + + + + + + ); + } +); export const AppRoutes = memo(() => { const { modal, setModal } = useUrlModal(); diff --git a/x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.component.tsx b/x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.component.tsx new file mode 100644 index 0000000000000..1fa673890fa82 --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.component.tsx @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import styled from 'styled-components'; + +import { + EuiPopover, + EuiText, + EuiForm, + EuiFormRow, + EuiFieldText, + EuiCopy, + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiButton, + EuiLink, + EuiHeaderLink, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +export interface Props { + endpointUrl: string; + cloudId: string; + managementUrl?: string; + learnMoreUrl: string; +} + +const Description = styled(EuiText)` + margin-bottom: ${({ theme }) => theme.eui.euiSizeL}; +`; + +export const DeploymentDetails = ({ endpointUrl, cloudId, learnMoreUrl, managementUrl }: Props) => { + const [isOpen, setIsOpen] = React.useState(false); + + const button = ( + setIsOpen(!isOpen)} iconType="iInCircle" iconSide="left" isActive> + {i18n.translate('xpack.fleet.integrations.deploymentButton', { + defaultMessage: 'View deployment details', + })} + + ); + + const management = managementUrl ? ( + + + + Create and manage API keys + + + + Learn more + + + + + ) : null; + + return ( + setIsOpen(false)} + button={button} + anchorPosition="downCenter" + > +
+ + Send data to Elastic from your applications by referencing your deployment and + Elasticsearch information. + + + + + + + + + + {(copy) => ( + + )} + + + + + + + + + + + + {(copy) => ( + + )} + + + + + {management} + +
+ + ); +}; diff --git a/x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.stories.tsx b/x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.stories.tsx new file mode 100644 index 0000000000000..445bf471c0fe9 --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.stories.tsx @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import type { Meta } from '@storybook/react'; +import { EuiHeader } from '@elastic/eui'; + +import { DeploymentDetails as ConnectedComponent } from './deployment_details'; +import type { Props as PureComponentProps } from './deployment_details.component'; +import { DeploymentDetails as PureComponent } from './deployment_details.component'; + +export default { + title: 'Sections/EPM/Deployment Details', + description: '', + decorators: [ + (storyFn) => { + const sections = [{ items: [] }, { items: [storyFn()] }]; + return ; + }, + ], +} as Meta; + +export const DeploymentDetails = () => { + return ; +}; + +DeploymentDetails.args = { + isCloudEnabled: true, +}; + +DeploymentDetails.argTypes = { + isCloudEnabled: { + type: { + name: 'boolean', + }, + defaultValue: true, + control: { + type: 'boolean', + }, + }, +}; + +export const Component = (props: PureComponentProps) => { + return ; +}; + +Component.args = { + cloudId: 'cloud-id', + endpointUrl: 'https://endpoint-url', + learnMoreUrl: 'https://learn-more-url', + managementUrl: 'https://management-url', +}; diff --git a/x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.tsx b/x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.tsx new file mode 100644 index 0000000000000..48c8fa56fb91b --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/components/header/deployment_details.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { useStartServices } from '../../hooks'; + +import { DeploymentDetails as Component } from './deployment_details.component'; + +export const DeploymentDetails = () => { + const { share, cloud, docLinks } = useStartServices(); + + // If the cloud plugin isn't enabled, we can't display the flyout. + if (!cloud) { + return null; + } + + const { isCloudEnabled, cloudId, cname } = cloud; + + // If cloud isn't enabled, we don't have a cloudId or a cname, we can't display the flyout. + if (!isCloudEnabled || !cloudId || !cname) { + return null; + } + + // If the cname doesn't start with a known prefix, we can't display the flyout. + // TODO: dover - this is a short term solution, see https://github.com/elastic/kibana/pull/114287#issuecomment-940111026 + if ( + !( + cname.endsWith('elastic-cloud.com') || + cname.endsWith('found.io') || + cname.endsWith('found.no') + ) + ) { + return null; + } + + const cnameNormalized = cname.startsWith('.') ? cname.substring(1) : cname; + const endpointUrl = `https://${cloudId}.${cnameNormalized}`; + + const managementUrl = share.url.locators + .get('MANAGEMENT_APP_LOCATOR') + ?.useUrl({ sectionId: 'security', appId: 'api_keys' }); + + const learnMoreUrl = docLinks.links.fleet.apiKeysLearnMore; + + return ; +}; diff --git a/x-pack/plugins/fleet/public/applications/integrations/components/header/header.tsx b/x-pack/plugins/fleet/public/applications/integrations/components/header/header.tsx new file mode 100644 index 0000000000000..e87c63e98ef28 --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/components/header/header.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiHeaderSectionItem, EuiHeaderSection, EuiHeaderLinks } from '@elastic/eui'; + +import type { AppMountParameters } from 'kibana/public'; + +import { HeaderPortal } from './header_portal'; +import { DeploymentDetails } from './deployment_details'; + +export const IntegrationsHeader = ({ + setHeaderActionMenu, +}: { + setHeaderActionMenu: AppMountParameters['setHeaderActionMenu']; +}) => { + return ( + + + + + + + + + + ); +}; diff --git a/x-pack/plugins/fleet/public/applications/integrations/components/header/header_portal.tsx b/x-pack/plugins/fleet/public/applications/integrations/components/header/header_portal.tsx new file mode 100644 index 0000000000000..d3dbbcf9628ec --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/components/header/header_portal.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AppMountParameters } from 'kibana/public'; +import type { FC } from 'react'; +import React, { useEffect, useMemo } from 'react'; +import { createPortalNode, InPortal, OutPortal } from 'react-reverse-portal'; + +import { toMountPoint } from '../../../../../../../../src/plugins/kibana_react/public'; + +export interface Props { + setHeaderActionMenu: AppMountParameters['setHeaderActionMenu']; +} + +export const HeaderPortal: FC = ({ children, setHeaderActionMenu }) => { + const portalNode = useMemo(() => createPortalNode(), []); + + useEffect(() => { + setHeaderActionMenu((element) => { + const mount = toMountPoint(); + return mount(element); + }); + + return () => { + portalNode.unmount(); + setHeaderActionMenu(undefined); + }; + }, [portalNode, setHeaderActionMenu]); + + return {children}; +}; diff --git a/x-pack/plugins/fleet/public/applications/integrations/components/header/index.ts b/x-pack/plugins/fleet/public/applications/integrations/components/header/index.ts new file mode 100644 index 0000000000000..e0a342326d972 --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/components/header/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { IntegrationsHeader } from './header'; diff --git a/x-pack/plugins/fleet/public/applications/integrations/index.tsx b/x-pack/plugins/fleet/public/applications/integrations/index.tsx index da8959a019ce5..0abb78f850076 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/index.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/index.tsx @@ -37,6 +37,7 @@ interface IntegrationsAppProps { history: AppMountParameters['history']; kibanaVersion: string; extensions: UIExtensionsStorage; + setHeaderActionMenu: AppMountParameters['setHeaderActionMenu']; } const IntegrationsApp = ({ basepath, @@ -45,6 +46,7 @@ const IntegrationsApp = ({ history, kibanaVersion, extensions, + setHeaderActionMenu, }: IntegrationsAppProps) => { return ( @@ -64,7 +67,7 @@ const IntegrationsApp = ({ export function renderApp( startServices: FleetStartServices, - { element, appBasePath, history }: AppMountParameters, + { element, appBasePath, history, setHeaderActionMenu }: AppMountParameters, config: FleetConfigType, kibanaVersion: string, extensions: UIExtensionsStorage @@ -77,6 +80,7 @@ export function renderApp( history={history} kibanaVersion={kibanaVersion} extensions={extensions} + setHeaderActionMenu={setHeaderActionMenu} />, element ); diff --git a/x-pack/plugins/fleet/public/mock/create_test_renderer.tsx b/x-pack/plugins/fleet/public/mock/create_test_renderer.tsx index d0724545ee902..a0a9ef405540a 100644 --- a/x-pack/plugins/fleet/public/mock/create_test_renderer.tsx +++ b/x-pack/plugins/fleet/public/mock/create_test_renderer.tsx @@ -41,6 +41,7 @@ export interface TestRenderer { kibanaVersion: string; AppWrapper: React.FC; render: UiRender; + setHeaderActionMenu: Function; } export const createFleetTestRendererMock = (): TestRenderer => { @@ -55,6 +56,7 @@ export const createFleetTestRendererMock = (): TestRenderer => { config: createConfigurationMock(), startInterface: createStartMock(extensions), kibanaVersion: '8.0.0', + setHeaderActionMenu: jest.fn(), AppWrapper: memo(({ children }) => { return ( { config: createConfigurationMock(), startInterface: createStartMock(extensions), kibanaVersion: '8.0.0', + setHeaderActionMenu: jest.fn(), AppWrapper: memo(({ children }) => { return ( { kibanaVersion={testRendererMocks.kibanaVersion} extensions={extensions} routerHistory={testRendererMocks.history} + setHeaderActionMenu={() => {}} > {children} diff --git a/x-pack/plugins/fleet/public/mock/plugin_dependencies.ts b/x-pack/plugins/fleet/public/mock/plugin_dependencies.ts index f78fe58a6ad88..0bf0213905e72 100644 --- a/x-pack/plugins/fleet/public/mock/plugin_dependencies.ts +++ b/x-pack/plugins/fleet/public/mock/plugin_dependencies.ts @@ -10,6 +10,7 @@ import { licensingMock } from '../../../licensing/public/mocks'; import { homePluginMock } from '../../../../../src/plugins/home/public/mocks'; import { navigationPluginMock } from '../../../../../src/plugins/navigation/public/mocks'; import { customIntegrationsMock } from '../../../../../src/plugins/custom_integrations/public/mocks'; +import { sharePluginMock } from '../../../../../src/plugins/share/public/mocks'; import type { MockedFleetSetupDeps, MockedFleetStartDeps } from './types'; @@ -27,5 +28,6 @@ export const createStartDepsMock = (): MockedFleetStartDeps => { data: dataPluginMock.createStartContract(), navigation: navigationPluginMock.createStartContract(), customIntegrations: customIntegrationsMock.createStart(), + share: sharePluginMock.createStartContract(), }; }; diff --git a/x-pack/plugins/fleet/public/plugin.ts b/x-pack/plugins/fleet/public/plugin.ts index d23bfcfe7b888..e1f263b0763e8 100644 --- a/x-pack/plugins/fleet/public/plugin.ts +++ b/x-pack/plugins/fleet/public/plugin.ts @@ -21,6 +21,8 @@ import type { CustomIntegrationsSetup, } from 'src/plugins/custom_integrations/public'; +import type { SharePluginStart } from 'src/plugins/share/public'; + import { DEFAULT_APP_CATEGORIES, AppNavLinkStatus } from '../../../../src/core/public'; import type { @@ -81,10 +83,12 @@ export interface FleetStartDeps { data: DataPublicPluginStart; navigation: NavigationPublicPluginStart; customIntegrations: CustomIntegrationsStart; + share: SharePluginStart; } export interface FleetStartServices extends CoreStart, FleetStartDeps { storage: Storage; + share: SharePluginStart; cloud?: CloudSetup; } @@ -134,6 +138,7 @@ export class FleetPlugin implements Plugin { + const cloud: CloudSetup = { + isCloudEnabled, + baseUrl: 'https://base.url', + cloudId: 'cloud-id', + cname: 'found.io', + deploymentUrl: 'https://deployment.url', + organizationUrl: 'https://organization.url', + profileUrl: 'https://profile.url', + snapshotsUrl: 'https://snapshots.url', + }; + + return cloud; +}; diff --git a/x-pack/plugins/fleet/storybook/context/doc_links.ts b/x-pack/plugins/fleet/storybook/context/doc_links.ts index 9b86ea03549f3..bb6d8086d1cd8 100644 --- a/x-pack/plugins/fleet/storybook/context/doc_links.ts +++ b/x-pack/plugins/fleet/storybook/context/doc_links.ts @@ -15,6 +15,7 @@ export const getDocLinks = () => { fleet: { learnMoreBlog: 'https://www.elastic.co/blog/elastic-agent-and-fleet-make-it-easier-to-integrate-your-systems-with-elastic', + apiKeysLearnMore: 'https://www.elastic.co/guide/en/kibana/master/api-keys.html', }, }, } as unknown as DocLinksStart; diff --git a/x-pack/plugins/fleet/storybook/context/index.tsx b/x-pack/plugins/fleet/storybook/context/index.tsx index c9c8e0be5d883..e6c0726e755c2 100644 --- a/x-pack/plugins/fleet/storybook/context/index.tsx +++ b/x-pack/plugins/fleet/storybook/context/index.tsx @@ -28,6 +28,8 @@ import { getUiSettings } from './ui_settings'; import { getNotifications } from './notifications'; import { stubbedStartServices } from './stubs'; import { getDocLinks } from './doc_links'; +import { getCloud } from './cloud'; +import { getShare } from './share'; // TODO: clintandrewhall - this is not ideal, or complete. The root context of Fleet applications // requires full start contracts of its dependencies. As a result, we have to mock all of those contracts @@ -36,6 +38,7 @@ import { getDocLinks } from './doc_links'; // // Expect this to grow as components that are given Stories need access to mocked services. export const StorybookContext: React.FC<{ storyContext?: StoryContext }> = ({ + storyContext, children: storyChildren, }) => { const basepath = ''; @@ -46,10 +49,12 @@ export const StorybookContext: React.FC<{ storyContext?: StoryContext }> = ({ ...stubbedStartServices, application: getApplication(), chrome: getChrome(), + cloud: getCloud({ isCloudEnabled: storyContext?.args.isCloudEnabled }), + customIntegrations: { + ContextProvider: getStorybookContextProvider(), + }, docLinks: getDocLinks(), http: getHttp(), - notifications: getNotifications(), - uiSettings: getUiSettings(), i18n: { Context: function I18nContext({ children }) { return {children}; @@ -58,9 +63,9 @@ export const StorybookContext: React.FC<{ storyContext?: StoryContext }> = ({ injectedMetadata: { getInjectedVar: () => null, }, - customIntegrations: { - ContextProvider: getStorybookContextProvider(), - }, + notifications: getNotifications(), + share: getShare(), + uiSettings: getUiSettings(), }; setHttpClient(startServices.http); @@ -81,12 +86,20 @@ export const StorybookContext: React.FC<{ storyContext?: StoryContext }> = ({ } as unknown as FleetConfigType; const extensions = {}; - const kibanaVersion = '1.2.3'; + const setHeaderActionMenu = () => {}; return ( {storyChildren} diff --git a/x-pack/plugins/fleet/storybook/context/share.ts b/x-pack/plugins/fleet/storybook/context/share.ts new file mode 100644 index 0000000000000..b6a8dfbcdcb67 --- /dev/null +++ b/x-pack/plugins/fleet/storybook/context/share.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SharePluginStart } from 'src/plugins/share/public'; + +export const getShare = () => { + const share: SharePluginStart = { + url: { + locators: { + get: () => ({ + useUrl: () => 'https://locator.url', + }), + }, + }, + } as unknown as SharePluginStart; + + return share; +}; diff --git a/x-pack/plugins/fleet/storybook/context/stubs.tsx b/x-pack/plugins/fleet/storybook/context/stubs.tsx index a7db4bd8f68cd..f72b176bd8d7b 100644 --- a/x-pack/plugins/fleet/storybook/context/stubs.tsx +++ b/x-pack/plugins/fleet/storybook/context/stubs.tsx @@ -14,8 +14,7 @@ type Stubs = | 'fatalErrors' | 'navigation' | 'overlays' - | 'savedObjects' - | 'cloud'; + | 'savedObjects'; type StubbedStartServices = Pick; @@ -27,5 +26,4 @@ export const stubbedStartServices: StubbedStartServices = { navigation: {} as FleetStartServices['navigation'], overlays: {} as FleetStartServices['overlays'], savedObjects: {} as FleetStartServices['savedObjects'], - cloud: {} as FleetStartServices['cloud'], }; diff --git a/x-pack/plugins/fleet/storybook/decorator.tsx b/x-pack/plugins/fleet/storybook/decorator.tsx index 8e68249809574..93870809c94bb 100644 --- a/x-pack/plugins/fleet/storybook/decorator.tsx +++ b/x-pack/plugins/fleet/storybook/decorator.tsx @@ -11,5 +11,5 @@ import type { DecoratorFn } from '@storybook/react'; import { StorybookContext } from './context'; export const decorator: DecoratorFn = (story, storybook) => { - return {story()}; + return {story()}; }; From 5de36a8229e563a1920bef28b37c49a4231b4098 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 13 Oct 2021 12:50:35 -0500 Subject: [PATCH 10/35] [kbn/optimizer] fix --update-limit docs (#114840) Co-authored-by: spalger --- packages/kbn-optimizer/src/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-optimizer/src/cli.ts b/packages/kbn-optimizer/src/cli.ts index 7f0c39ccd0e55..41ca656259fc6 100644 --- a/packages/kbn-optimizer/src/cli.ts +++ b/packages/kbn-optimizer/src/cli.ts @@ -207,7 +207,7 @@ export function runKbnOptimizerCli(options: { defaultLimitsPath: string }) { --no-inspect-workers when inspecting the parent process, don't inspect the workers --limits path to a limits.yml file to read, defaults to $KBN_OPTIMIZER_LIMITS_PATH or source file --validate-limits validate the limits.yml config to ensure that there are limits defined for every bundle - --update-limits run a build and rewrite the limits file to include the current bundle sizes +5kb + --update-limits run a build and rewrite the limits file to include the current bundle sizes +15kb `, }, } From 6d24de9d6eaf95d115a250e70eacf3ae7d1e76f6 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 13 Oct 2021 12:00:15 -0600 Subject: [PATCH 11/35] [Stack Monitoring] Fix shard size alerts (#114357) * [Stack Monitoring] Fix shard size alerts * Removing the source filter for source_node.* * Removing superfluous types * Removing superfluous nodeId and nodeName from test --- x-pack/plugins/monitoring/common/types/alerts.ts | 4 +++- .../server/alerts/large_shard_size_rule.test.ts | 4 ---- .../server/lib/alerts/fetch_index_shard_size.ts | 15 +++------------ 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/monitoring/common/types/alerts.ts b/x-pack/plugins/monitoring/common/types/alerts.ts index bbd217169469d..9abca4cbdc948 100644 --- a/x-pack/plugins/monitoring/common/types/alerts.ts +++ b/x-pack/plugins/monitoring/common/types/alerts.ts @@ -208,9 +208,11 @@ export interface CCRReadExceptionsUIMeta extends CCRReadExceptionsStats { itemLabel: string; } -export interface IndexShardSizeStats extends AlertNodeStats { +export interface IndexShardSizeStats { shardIndex: string; shardSize: number; + clusterUuid: string; + ccs?: string; } export interface IndexShardSizeUIMeta extends IndexShardSizeStats { diff --git a/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.test.ts index 0b8509c4fa56a..f7d6081edd306 100644 --- a/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.test.ts @@ -85,14 +85,10 @@ describe('LargeShardSizeRule', () => { const shardSize = 0; const clusterUuid = 'abc123'; const clusterName = 'testCluster'; - const nodeId = 'myNodeId'; - const nodeName = 'myNodeName'; const stat = { shardIndex, shardSize, clusterUuid, - nodeId, - nodeName, }; const replaceState = jest.fn(); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts index 98bb546b43ab9..9259adc63e546 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts @@ -11,12 +11,8 @@ import { ElasticsearchIndexStats, ElasticsearchResponseHit } from '../../../comm import { ESGlobPatterns, RegExPatterns } from '../../../common/es_glob_patterns'; import { Globals } from '../../static_globals'; -interface SourceNode { - name: string; - uuid: string; -} type TopHitType = ElasticsearchResponseHit & { - _source: { index_stats?: Partial; source_node?: SourceNode }; + _source: { index_stats?: Partial }; }; const memoizedIndexPatterns = (globPatterns: string) => { @@ -90,8 +86,6 @@ export async function fetchIndexShardSize( '_index', 'index_stats.shards.primaries', 'index_stats.primaries.store.size_in_bytes', - 'source_node.name', - 'source_node.uuid', ], }, size: 1, @@ -135,10 +129,10 @@ export async function fetchIndexShardSize( } const { _index: monitoringIndexName, - _source: { source_node: sourceNode, index_stats: indexStats }, + _source: { index_stats: indexStats }, } = topHit; - if (!indexStats || !indexStats.primaries || !sourceNode) { + if (!indexStats || !indexStats.primaries) { continue; } @@ -151,7 +145,6 @@ export async function fetchIndexShardSize( * We can only calculate the average primary shard size at this point, since we don't have * data (in .monitoring-es* indices) to give us individual shards. This might change in the future */ - const { name: nodeName, uuid: nodeId } = sourceNode; const avgShardSize = primaryShardSizeBytes / totalPrimaryShards; if (avgShardSize < thresholdBytes) { continue; @@ -161,8 +154,6 @@ export async function fetchIndexShardSize( shardIndex, shardSize, clusterUuid, - nodeName, - nodeId, ccs: monitoringIndexName.includes(':') ? monitoringIndexName.split(':')[0] : undefined, }); } From feed7391a0cd0909e1eefb1953c4a8fc81328118 Mon Sep 17 00:00:00 2001 From: Stacey Gammon Date: Wed, 13 Oct 2021 14:01:46 -0400 Subject: [PATCH 12/35] Update kibana_platform_plugin_intro with more details on packages vs plugins (#114713) * Update kibana_platform_plugin_intro.mdx * updates * Update kibana_platform_plugin_intro.mdx * Update kibana_platform_plugin_intro.mdx * Update kibana_platform_plugin_intro.mdx * Update dev_docs/key_concepts/kibana_platform_plugin_intro.mdx Co-authored-by: Tyler Smalley * Update dev_docs/key_concepts/kibana_platform_plugin_intro.mdx Co-authored-by: Tyler Smalley * Update dev_docs/key_concepts/kibana_platform_plugin_intro.mdx Co-authored-by: Tyler Smalley * Update dev_docs/key_concepts/kibana_platform_plugin_intro.mdx Co-authored-by: Brandon Kobel * Update dev_docs/key_concepts/kibana_platform_plugin_intro.mdx Co-authored-by: Brandon Kobel Co-authored-by: Tyler Smalley Co-authored-by: Brandon Kobel --- .../kibana_platform_plugin_intro.mdx | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/dev_docs/key_concepts/kibana_platform_plugin_intro.mdx b/dev_docs/key_concepts/kibana_platform_plugin_intro.mdx index 133b96f44da88..737b9d8708f29 100644 --- a/dev_docs/key_concepts/kibana_platform_plugin_intro.mdx +++ b/dev_docs/key_concepts/kibana_platform_plugin_intro.mdx @@ -33,13 +33,28 @@ At a super high-level, Kibana is composed of **plugins**, **core**, and **Kibana -If it's stateful, it has to go in a plugin, but packages are often a good choices for stateless utilities. Stateless code exported publicly from a plugin will increase the page load bundle size of _every single page_, even if none of those plugin's services are actually needed. With packages, however, only code that is needed for the current page is downloaded. +When the [Bazel migration](https://github.com/elastic/kibana/blob/master/legacy_rfcs/text/0015_bazel.md) is complete, all code, including plugins, will be a package. With that, packages won't be required to be in the `packages/` directory and can be located somewhere that makes more sense structurally. -The downside however is that the packages folder is far away from the plugins folder so having a part of your code in a plugin and the rest in a package may make it hard to find, leading to duplication. +In the meantime, the following can be used to determine whether it makes sense to add code to a package inside the `packages` folder, or a plugin inside `src/plugins` or `x-pack/plugins`. -The Operations team hopes to resolve this conundrum by supporting co-located packages and plugins and automatically putting all stateless code inside a package. You can track this work by following [this issue](https://github.com/elastic/kibana/issues/112886). -Until then, consider whether it makes sense to logically separate the code, and consider the size of the exports, when determining whether you should put stateless public exports in a package or a plugin. +**If the code is stateful, it has to be exposed from a plugin's . Do not statically export stateful code.** + +Benefits to packages: + +1. _Potentially_ reduced page load time. All code that is statically exported from plugins will be downloaded on _every single page load_, even if that code isn't needed. With packages, only code that is imported is downloaded, which can be minimized by using async imports. +2. Puts the consumer is in charge of how and when to async import. If a consumer async imports code exported from a plugin, it makes no difference, because of the above point. It's already been downloaded. However, simply moving code into a package is _not_ a guaranteed performance improvement. It does give the consumer the power to make smart performance choices, however. If they require code from multiple packages, the consumer can async import from multiple packages at the same time. Read more in our . + +Downsides to packages: + +1. It's not . The packages folder is far away from the plugins folder. Having your stateless code in a plugin and the rest in a package may make it hard to find, leading to duplication. The Operations team hopes to fix this by supporting packages and plugins existing in the same folder. You can track this work by following [this issue](https://github.com/elastic/kibana/issues/112886). + +2. Development overhead. Developers have to run `yarn kbn watch` to have changes rebuilt automatically. [Phase II](https://github.com/elastic/kibana/blob/master/legacy_rfcs/text/0015_bazel.md#phase-ii---docs-developer-experience) of the Bazel migration work will bring the development experience on par with plugin development. This work can be tracked [here](https://github.com/elastic/kibana/issues/104519). + +3. Development performance. Rebuild time is typically longer than it would be for the same code in a plugin. The reasons are captured in [this issue](https://github.com/elastic/kibana/issues/107648). The ops team is actively working to reduce this performance increase. + + +As you can see, the answer to "Should I put my code in a plugin or a package" is 'It Depends'. If you are still having a hard time determining what the best path location is, reach out to the Kibana Operations Team (#kibana-operations) for help. From dab5a59fc294061dd1a8a35811573d09285741cb Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 13 Oct 2021 18:25:23 +0000 Subject: [PATCH 13/35] skip suite failing es promotion (#114885) --- x-pack/test/security_solution_endpoint_api_int/apis/package.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/package.ts b/x-pack/test/security_solution_endpoint_api_int/apis/package.ts index a8fd5a612b306..fdacc07426871 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/package.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/package.ts @@ -66,7 +66,8 @@ export default function ({ getService }: FtrProviderContext) { }); }; - describe('Endpoint package', () => { + // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/114885 + describe.skip('Endpoint package', () => { describe('network processors', () => { let networkIndexData: InsertedEvents; From 21f45283be5712b494ed60f0630b2828c2807e9a Mon Sep 17 00:00:00 2001 From: Kaarina Tungseth Date: Wed, 13 Oct 2021 13:29:50 -0500 Subject: [PATCH 14/35] [DOCS] Documents monitoring.cluster_alerts.allowedSpaces (#114669) * [DOCS] Documents monitoring.cluster_alerts.allowedSpaces * Update docs/settings/spaces-settings.asciidoc Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> --- docs/settings/spaces-settings.asciidoc | 29 ++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/settings/spaces-settings.asciidoc b/docs/settings/spaces-settings.asciidoc index 30b7beceb70ba..969adb93185d0 100644 --- a/docs/settings/spaces-settings.asciidoc +++ b/docs/settings/spaces-settings.asciidoc @@ -5,20 +5,23 @@ Spaces settings ++++ -By default, Spaces is enabled in Kibana, and you can secure Spaces using -roles when Security is enabled. - -[float] -[[spaces-settings]] -==== Spaces settings +By default, spaces is enabled in {kib}. To secure spaces, <>. `xpack.spaces.enabled`:: -deprecated:[7.16.0,"In 8.0 and later, this setting will no longer be supported."] -Set to `true` (default) to enable Spaces in {kib}. -This setting is deprecated. Starting in 8.0, it will not be possible to disable this plugin. +deprecated:[7.16.0,"In 8.0 and later, this setting will no longer be supported and it will not be possible to disable this plugin."] +To enable spaces, set to `true`. +The default is `true`. `xpack.spaces.maxSpaces`:: -The maximum amount of Spaces that can be used with this instance of {kib}. Some operations -in {kib} return all spaces using a single `_search` from {es}, so this must be -set lower than the `index.max_result_window` in {es}. -Defaults to `1000`. +The maximum number of spaces that you can use with the {kib} instance. Some {kib} operations +return all spaces using a single `_search` from {es}, so you must +configure this setting lower than the `index.max_result_window` in {es}. +The default is `1000`. + +`monitoring.cluster_alerts-allowedSpaces` {ess-icon}:: +Specifies the spaces where cluster alerts are automatically generated. +You must specify all spaces where you want to generate alerts, including the default space. +When the default space is unspecified, {kib} is unable to generate an alert for the default space. +{es} clusters that run on {es} services are all containers. To send monitoring data +from your self-managed {es} installation to {es} services, set to `false`. +The default is `true`. From 95cd74d7fa744afec65e9fe72ceaf7abda30e262 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 13 Oct 2021 19:35:51 +0100 Subject: [PATCH 15/35] [ML] Using data views service for loading data views (#113961) * [ML] Using data views service for loading data views * removing more saved object client uses * removing IIndexPattern use * removing IndexPattern use * removing more depricated types * fixing teste * fixing index pattern loading * tiny refactor * fixing rollup index test * changes based on review * adding size to find calls Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/ml/common/types/kibana.ts | 2 - x-pack/plugins/ml/kibana.json | 1 + x-pack/plugins/ml/public/application/app.tsx | 1 + .../components/data_grid/common.ts | 17 +- .../data_recognizer/data_recognizer.d.ts | 4 +- .../full_time_range_selector.test.tsx | 10 +- .../full_time_range_selector.tsx | 7 +- .../full_time_range_selector_service.ts | 6 +- .../scatterplot_matrix/scatterplot_matrix.tsx | 4 +- .../use_scatterplot_field_options.ts | 4 +- .../contexts/ml/__mocks__/index_pattern.ts | 4 +- .../contexts/ml/__mocks__/index_patterns.ts | 4 +- .../application/contexts/ml/ml_context.ts | 6 +- .../common/use_results_view_config.ts | 6 +- .../hooks/use_index_data.ts | 7 +- .../expandable_section_results.tsx | 4 +- .../exploration_query_bar.tsx | 4 +- .../exploration_results_table.tsx | 4 +- .../use_exploration_results.ts | 4 +- .../outlier_exploration/use_outlier_data.ts | 4 +- .../action_clone/clone_action_name.tsx | 21 +-- .../action_delete/delete_action_name.test.tsx | 2 +- .../action_delete/use_delete_action.tsx | 21 +-- .../source_selection.test.tsx | 1 + .../index_based/data_loader/data_loader.ts | 9 +- .../explorer_query_bar/explorer_query_bar.tsx | 6 +- .../custom_url_editor/editor.test.tsx | 4 +- .../components/custom_url_editor/editor.tsx | 4 +- .../components/custom_url_editor/utils.d.ts | 4 +- .../edit_job_flyout/edit_utils.d.ts | 4 +- .../components/edit_job_flyout/edit_utils.js | 28 +-- .../edit_job_flyout/tabs/custom_urls.tsx | 4 +- .../common/chart_loader/chart_loader.ts | 4 +- .../new_job/common/index_pattern_context.ts | 4 +- .../job_creator/advanced_job_creator.ts | 8 +- .../job_creator/categorization_job_creator.ts | 8 +- .../new_job/common/job_creator/job_creator.ts | 10 +- .../common/job_creator/job_creator_factory.ts | 4 +- .../job_creator/multi_metric_job_creator.ts | 8 +- .../job_creator/population_job_creator.ts | 8 +- .../common/job_creator/rare_job_creator.ts | 8 +- .../job_creator/single_metric_job_creator.ts | 8 +- .../categorization_examples_loader.ts | 4 +- .../preconfigured_job_redirect.ts | 6 +- .../jobs/new_job/utils/new_job_utils.test.ts | 4 +- .../public/application/routing/resolvers.ts | 4 +- .../ml/public/application/routing/router.tsx | 12 +- .../services/field_format_service.ts | 8 +- .../load_new_job_capabilities.ts | 6 +- .../new_job_capabilities._service.test.ts | 4 +- .../new_job_capabilities_service.ts | 5 +- .../new_job_capabilities_service_analytics.ts | 5 +- .../remove_nested_field_children.test.ts | 4 +- .../application/util/dependency_cache.ts | 15 +- .../util/field_types_utils.test.ts | 19 +- .../application/util/field_types_utils.ts | 5 +- .../ml/public/application/util/index_utils.ts | 50 ++---- .../anomaly_charts_embeddable.tsx | 4 +- x-pack/plugins/ml/public/embeddables/types.ts | 4 +- .../plugins/ml/server/lib/data_views_utils.ts | 26 +++ x-pack/plugins/ml/server/lib/route_guard.ts | 23 ++- .../data_frame_analytics/index_patterns.ts | 22 +-- .../data_recognizer/data_recognizer.test.ts | 12 +- .../models/data_recognizer/data_recognizer.ts | 62 ++++--- .../ml/server/models/data_recognizer/index.ts | 2 +- .../data_view_rollup_cloudwatch.json | 151 ++++++++++++++++ .../responses/kibana_saved_objects.json | 35 ---- .../job_service/new_job_caps/field_service.ts | 20 +-- .../new_job_caps/new_job_caps.test.ts | 16 +- .../job_service/new_job_caps/new_job_caps.ts | 9 +- .../models/job_service/new_job_caps/rollup.ts | 44 ++--- x-pack/plugins/ml/server/plugin.ts | 22 ++- .../ml/server/routes/data_frame_analytics.ts | 170 +++++++++--------- .../plugins/ml/server/routes/job_service.ts | 35 ++-- x-pack/plugins/ml/server/routes/modules.ts | 80 +++++++-- .../shared_services/providers/modules.ts | 42 ++--- .../server/shared_services/shared_services.ts | 36 +++- x-pack/plugins/ml/server/types.ts | 2 + x-pack/plugins/ml/tsconfig.json | 1 + 79 files changed, 721 insertions(+), 529 deletions(-) create mode 100644 x-pack/plugins/ml/server/lib/data_views_utils.ts create mode 100644 x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/responses/data_view_rollup_cloudwatch.json delete mode 100644 x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/responses/kibana_saved_objects.json diff --git a/x-pack/plugins/ml/common/types/kibana.ts b/x-pack/plugins/ml/common/types/kibana.ts index 7783a02c2dd37..cc7b68b40149f 100644 --- a/x-pack/plugins/ml/common/types/kibana.ts +++ b/x-pack/plugins/ml/common/types/kibana.ts @@ -8,7 +8,6 @@ // custom edits or fixes for default kibana types which are incomplete import type { SimpleSavedObject } from 'kibana/public'; -import type { IndexPatternAttributes } from 'src/plugins/data/common'; import type { FieldFormatsRegistry } from '../../../../../src/plugins/field_formats/common'; export type IndexPatternTitle = string; @@ -18,7 +17,6 @@ export interface Route { k7Breadcrumbs: () => any; } -export type IndexPatternSavedObject = SimpleSavedObject; // TODO define saved object type export type SavedSearchSavedObject = SimpleSavedObject; diff --git a/x-pack/plugins/ml/kibana.json b/x-pack/plugins/ml/kibana.json index 310ac5d65c986..9ed05bbdc2edf 100644 --- a/x-pack/plugins/ml/kibana.json +++ b/x-pack/plugins/ml/kibana.json @@ -8,6 +8,7 @@ ], "requiredPlugins": [ "data", + "dataViews", "cloud", "features", "dataVisualizer", diff --git a/x-pack/plugins/ml/public/application/app.tsx b/x-pack/plugins/ml/public/application/app.tsx index c7e457c0b5e00..6259cecae78b5 100644 --- a/x-pack/plugins/ml/public/application/app.tsx +++ b/x-pack/plugins/ml/public/application/app.tsx @@ -135,6 +135,7 @@ export const renderApp = ( urlGenerators: deps.share.urlGenerators, maps: deps.maps, dataVisualizer: deps.dataVisualizer, + dataViews: deps.data.dataViews, }); appMountParams.onAppLeave((actions) => actions.default()); diff --git a/x-pack/plugins/ml/public/application/components/data_grid/common.ts b/x-pack/plugins/ml/public/application/components/data_grid/common.ts index a64594e86a757..6fc6f298e73d8 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/common.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/common.ts @@ -19,12 +19,9 @@ import { i18n } from '@kbn/i18n'; import { CoreSetup } from 'src/core/public'; -import { - IndexPattern, - IFieldType, - ES_FIELD_TYPES, - KBN_FIELD_TYPES, -} from '../../../../../../../src/plugins/data/public'; +import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; + +import type { DataView, DataViewField } from '../../../../../../../src/plugins/data_views/common'; import { DEFAULT_RESULTS_FIELD } from '../../../../common/constants/data_frame_analytics'; import { extractErrorMessage } from '../../../../common/util/errors'; @@ -72,7 +69,7 @@ export const euiDataGridToolbarSettings = { showFullScreenSelector: false, }; -export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): string[] => { +export const getFieldsFromKibanaIndexPattern = (indexPattern: DataView): string[] => { const allFields = indexPattern.fields.map((f) => f.name); const indexPatternFields: string[] = allFields.filter((f) => { if (indexPattern.metaFields.includes(f)) { @@ -98,7 +95,7 @@ export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): str * @param RuntimeMappings */ export function getCombinedRuntimeMappings( - indexPattern: IndexPattern | undefined, + indexPattern: DataView | undefined, runtimeMappings?: RuntimeMappings ): RuntimeMappings | undefined { let combinedRuntimeMappings = {}; @@ -219,7 +216,7 @@ export const getDataGridSchemaFromESFieldType = ( }; export const getDataGridSchemaFromKibanaFieldType = ( - field: IFieldType | undefined + field: DataViewField | undefined ): string | undefined => { // Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json'] // To fall back to the default string schema it needs to be undefined. @@ -312,7 +309,7 @@ export const getTopClasses = (row: Record, mlResultsField: string): }; export const useRenderCellValue = ( - indexPattern: IndexPattern | undefined, + indexPattern: DataView | undefined, pagination: IndexPagination, tableItems: DataGridItem[], resultsField?: string, diff --git a/x-pack/plugins/ml/public/application/components/data_recognizer/data_recognizer.d.ts b/x-pack/plugins/ml/public/application/components/data_recognizer/data_recognizer.d.ts index ff6363ea2cc6e..fb9648f5ef4af 100644 --- a/x-pack/plugins/ml/public/application/components/data_recognizer/data_recognizer.d.ts +++ b/x-pack/plugins/ml/public/application/components/data_recognizer/data_recognizer.d.ts @@ -7,10 +7,10 @@ import { FC } from 'react'; import { SavedSearchSavedObject } from '../../../../common/types/kibana'; -import type { IIndexPattern } from '../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../src/plugins/data_views/public'; declare const DataRecognizer: FC<{ - indexPattern: IIndexPattern; + indexPattern: DataView; savedSearch: SavedSearchSavedObject | null; results: { count: number; diff --git a/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector.test.tsx b/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector.test.tsx index f9f8d9a370bab..72641499fe6af 100644 --- a/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector.test.tsx +++ b/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector.test.tsx @@ -8,15 +8,15 @@ import React from 'react'; import { shallowWithIntl } from '@kbn/test/jest'; import { FullTimeRangeSelector } from './index'; -import { Query } from 'src/plugins/data/public'; -import { IndexPattern } from '../../../../../../../src/plugins/data/public'; +import type { Query } from 'src/plugins/data/public'; +import type { DataView } from '../../../../../../../src/plugins/data_views/public'; // Create a mock for the setFullTimeRange function in the service. // The mock is hoisted to the top, so need to prefix the mock function // with 'mock' so it can be used lazily. -const mockSetFullTimeRange = jest.fn((indexPattern: IndexPattern, query: Query) => true); +const mockSetFullTimeRange = jest.fn((indexPattern: DataView, query: Query) => true); jest.mock('./full_time_range_selector_service', () => ({ - setFullTimeRange: (indexPattern: IndexPattern, query: Query) => + setFullTimeRange: (indexPattern: DataView, query: Query) => mockSetFullTimeRange(indexPattern, query), })); @@ -26,7 +26,7 @@ describe('FullTimeRangeSelector', () => { fields: [], title: 'test-index-pattern', timeFieldName: '@timestamp', - } as unknown as IndexPattern; + } as unknown as DataView; const query: Query = { language: 'kuery', diff --git a/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector.tsx b/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector.tsx index f087754bd275b..3c9689c8c108b 100644 --- a/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector.tsx +++ b/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector.tsx @@ -8,12 +8,13 @@ import React, { FC } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; -import { Query, IndexPattern } from 'src/plugins/data/public'; +import type { Query } from 'src/plugins/data/public'; import { EuiButton } from '@elastic/eui'; +import type { DataView } from '../../../../../../../src/plugins/data_views/public'; import { setFullTimeRange } from './full_time_range_selector_service'; interface Props { - indexPattern: IndexPattern; + indexPattern: DataView; query: Query; disabled: boolean; callback?: (a: any) => void; @@ -23,7 +24,7 @@ interface Props { // to the time range of data in the index(es) mapped to the supplied Kibana index pattern or query. export const FullTimeRangeSelector: FC = ({ indexPattern, query, disabled, callback }) => { // wrapper around setFullTimeRange to allow for the calling of the optional callBack prop - async function setRange(i: IndexPattern, q: Query) { + async function setRange(i: DataView, q: Query) { const fullTimeRange = await setFullTimeRange(i, q); if (typeof callback === 'function') { callback(fullTimeRange); diff --git a/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector_service.ts b/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector_service.ts index 1e9c1e2c1b74d..8f0d344a36f36 100644 --- a/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector_service.ts +++ b/x-pack/plugins/ml/public/application/components/full_time_range_selector/full_time_range_selector_service.ts @@ -8,11 +8,11 @@ import moment from 'moment'; import { i18n } from '@kbn/i18n'; -import { Query } from 'src/plugins/data/public'; +import type { Query } from 'src/plugins/data/public'; import dateMath from '@elastic/datemath'; import { getTimefilter, getToastNotifications } from '../../util/dependency_cache'; import { ml, GetTimeFieldRangeResponse } from '../../services/ml_api_service'; -import { IndexPattern } from '../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../src/plugins/data_views/public'; import { isPopulatedObject } from '../../../../common/util/object_utils'; import { RuntimeMappings } from '../../../../common/types/fields'; @@ -22,7 +22,7 @@ export interface TimeRange { } export async function setFullTimeRange( - indexPattern: IndexPattern, + indexPattern: DataView, query: Query ): Promise { try { diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx index b83965b52befc..d64a180bfa8b6 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx @@ -24,7 +24,7 @@ import { import { i18n } from '@kbn/i18n'; -import { IndexPattern } from '../../../../../../../src/plugins/data/public'; +import { DataView } from '../../../../../../../src/plugins/data_views/public'; import { extractErrorMessage } from '../../../../common'; import { isRuntimeMappings } from '../../../../common/util/runtime_field_utils'; import { stringHash } from '../../../../common/util/string_utils'; @@ -89,7 +89,7 @@ export interface ScatterplotMatrixProps { legendType?: LegendType; searchQuery?: ResultsSearchQuery; runtimeMappings?: RuntimeMappings; - indexPattern?: IndexPattern; + indexPattern?: DataView; } export const ScatterplotMatrix: FC = ({ diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts index d8b5b23f68847..543ab0a0c0982 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts @@ -7,12 +7,12 @@ import { useMemo } from 'react'; -import type { IndexPattern } from '../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../src/plugins/data_views/public'; import { ML__INCREMENTAL_ID } from '../../data_frame_analytics/common/fields'; export const useScatterplotFieldOptions = ( - indexPattern?: IndexPattern, + indexPattern?: DataView, includes?: string[], excludes?: string[], resultsField = '' diff --git a/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/index_pattern.ts b/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/index_pattern.ts index 9d53efad86d38..93f92002c4bfd 100644 --- a/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/index_pattern.ts +++ b/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/index_pattern.ts @@ -5,10 +5,10 @@ * 2.0. */ -import { IndexPattern } from '../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../src/plugins/data_views/public'; export const indexPatternMock = { id: 'the-index-pattern-id', title: 'the-index-pattern-title', fields: [], -} as unknown as IndexPattern; +} as unknown as DataView; diff --git a/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/index_patterns.ts b/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/index_patterns.ts index 7dfbcf1675692..571ce8ac3f423 100644 --- a/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/index_patterns.ts +++ b/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/index_patterns.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IndexPatternsContract } from '../../../../../../../../src/plugins/data/public'; +import type { DataViewsContract } from '../../../../../../../../src/plugins/data_views/public'; export const indexPatternsMock = new (class { fieldFormats = []; @@ -19,4 +19,4 @@ export const indexPatternsMock = new (class { getIds = jest.fn(); getTitles = jest.fn(); make = jest.fn(); -})() as unknown as IndexPatternsContract; +})() as unknown as DataViewsContract; diff --git a/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts b/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts index 02c0945cf998f..cd7059b5302f2 100644 --- a/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts +++ b/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts @@ -6,15 +6,15 @@ */ import React from 'react'; -import { IndexPattern, IndexPatternsContract } from '../../../../../../../src/plugins/data/public'; +import { DataView, DataViewsContract } from '../../../../../../../src/plugins/data_views/public'; import { SavedSearchSavedObject } from '../../../../common/types/kibana'; import { MlServicesContext } from '../../app'; export interface MlContextValue { combinedQuery: any; - currentIndexPattern: IndexPattern; // TODO this should be IndexPattern or null + currentIndexPattern: DataView; // TODO this should be IndexPattern or null currentSavedSearch: SavedSearchSavedObject | null; - indexPatterns: IndexPatternsContract; + indexPatterns: DataViewsContract; kibanaConfig: any; // IUiSettingsClient; kibanaVersion: string; } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts index e1f0db4e9291c..a235885a9a5b7 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts @@ -9,7 +9,7 @@ import { useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; -import { IndexPattern } from '../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../src/plugins/data_views/public'; import { extractErrorMessage } from '../../../../common/util/errors'; @@ -34,7 +34,7 @@ export const useResultsViewConfig = (jobId: string) => { const mlContext = useMlContext(); const trainedModelsApiService = useTrainedModelsApiService(); - const [indexPattern, setIndexPattern] = useState(undefined); + const [indexPattern, setIndexPattern] = useState(undefined); const [indexPatternErrorMessage, setIndexPatternErrorMessage] = useState( undefined ); @@ -99,7 +99,7 @@ export const useResultsViewConfig = (jobId: string) => { ? jobConfigUpdate.dest.index[0] : jobConfigUpdate.dest.index; const destIndexPatternId = getIndexPatternIdFromName(destIndex) || destIndex; - let indexP: IndexPattern | undefined; + let indexP: DataView | undefined; try { indexP = await mlContext.indexPatterns.get(destIndexPatternId); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts index 60a5a548c8621..f3779e1968985 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts @@ -11,7 +11,7 @@ import { estypes } from '@elastic/elasticsearch'; import { EuiDataGridColumn } from '@elastic/eui'; import { CoreSetup } from 'src/core/public'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; import { isRuntimeMappings } from '../../../../../../common/util/runtime_field_utils'; import { RuntimeMappings } from '../../../../../../common/types/fields'; import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../../../common/constants/field_histograms'; @@ -52,13 +52,14 @@ function getRuntimeFieldColumns(runtimeMappings: RuntimeMappings) { }); } -function getIndexPatternColumns(indexPattern: IndexPattern, fieldsFilter: string[]) { +function getIndexPatternColumns(indexPattern: DataView, fieldsFilter: string[]) { const { fields } = newJobCapsServiceAnalytics; return fields .filter((field) => fieldsFilter.includes(field.name)) .map((field) => { const schema = + // @ts-expect-error field is not DataViewField getDataGridSchemaFromESFieldType(field.type) || getDataGridSchemaFromKibanaFieldType(field); return { @@ -71,7 +72,7 @@ function getIndexPatternColumns(indexPattern: IndexPattern, fieldsFilter: string } export const useIndexData = ( - indexPattern: IndexPattern, + indexPattern: DataView, query: Record | undefined, toastNotifications: CoreSetup['notifications']['toasts'], runtimeMappings?: RuntimeMappings diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_results.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_results.tsx index d67473d9d3220..2c2df0cd3d905 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_results.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_results.tsx @@ -12,7 +12,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { EuiDataGridColumn, EuiSpacer, EuiText } from '@elastic/eui'; -import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../../src/plugins/data_views/public'; import { isClassificationAnalysis, @@ -104,7 +104,7 @@ const getResultsSectionHeaderItems = ( interface ExpandableSectionResultsProps { colorRange?: ReturnType; indexData: UseIndexDataReturnType; - indexPattern?: IndexPattern; + indexPattern?: DataView; jobConfig?: DataFrameAnalyticsConfig; needsDestIndexPattern: boolean; searchQuery: SavedSearchQuery; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx index 1a5f1bad997e2..3639836c6be01 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx @@ -13,7 +13,7 @@ import { debounce } from 'lodash'; import { fromKueryExpression, luceneStringToDsl, toElasticsearchQuery } from '@kbn/es-query'; import { estypes } from '@elastic/elasticsearch'; import { Dictionary } from '../../../../../../../common/types/common'; -import { IIndexPattern } from '../../../../../../../../../../src/plugins/data/common'; +import { DataView } from '../../../../../../../../../../src/plugins/data_views/common'; import { Query, QueryStringInput } from '../../../../../../../../../../src/plugins/data/public'; import { @@ -29,7 +29,7 @@ interface ErrorMessage { } export interface ExplorationQueryBarProps { - indexPattern: IIndexPattern; + indexPattern: DataView; setSearchQuery: (update: { queryString: string; query?: SavedSearchQuery; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx index 416d2f8b29d3b..41c434c7160cf 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx @@ -7,7 +7,7 @@ import React, { FC } from 'react'; -import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../../src/plugins/data_views/public'; import { getToastNotifications } from '../../../../../util/dependency_cache'; import { useMlKibana } from '../../../../../contexts/kibana'; @@ -22,7 +22,7 @@ import { ExpandableSectionResults } from '../expandable_section'; import { useExplorationResults } from './use_exploration_results'; interface Props { - indexPattern: IndexPattern; + indexPattern: DataView; jobConfig: DataFrameAnalyticsConfig; jobStatus?: DataFrameTaskStateType; needsDestIndexPattern: boolean; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts index 02e3f0abac4be..6e0d513a35b9a 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts @@ -13,7 +13,7 @@ import { CoreSetup } from 'src/core/public'; import { i18n } from '@kbn/i18n'; import { MlApiServices } from '../../../../../services/ml_api_service'; -import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../../src/plugins/data_views/public'; import { DataLoader } from '../../../../../datavisualizer/index_based/data_loader'; @@ -41,7 +41,7 @@ import { FeatureImportanceBaseline } from '../../../../../../../common/types/fea import { useExplorationDataGrid } from './use_exploration_data_grid'; export const useExplorationResults = ( - indexPattern: IndexPattern | undefined, + indexPattern: DataView | undefined, jobConfig: DataFrameAnalyticsConfig | undefined, searchQuery: SavedSearchQuery, toastNotifications: CoreSetup['notifications']['toasts'], diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts index d630fedc72d3f..d0f048ac02606 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts @@ -9,7 +9,7 @@ import { useEffect, useMemo } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; -import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../../src/plugins/data_views/public'; import { DataLoader } from '../../../../../datavisualizer/index_based/data_loader'; @@ -41,7 +41,7 @@ import { getFeatureCount, getOutlierScoreFieldName } from './common'; import { useExplorationDataGrid } from '../exploration_results_table/use_exploration_data_grid'; export const useOutlierData = ( - indexPattern: IndexPattern | undefined, + indexPattern: DataView | undefined, jobConfig: DataFrameAnalyticsConfig | undefined, searchQuery: SavedSearchQuery ): UseIndexDataReturnType => { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx index a3f18801b88f9..8ef743d2eea9f 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx @@ -9,7 +9,6 @@ import { EuiToolTip } from '@elastic/eui'; import React, { FC } from 'react'; import { cloneDeep, isEqual } from 'lodash'; import { i18n } from '@kbn/i18n'; -import { IIndexPattern } from 'src/plugins/data/common'; import { DeepReadonly } from '../../../../../../../common/types/common'; import { DataFrameAnalyticsConfig, isOutlierAnalysis } from '../../../../common'; import { isClassificationAnalysis, isRegressionAnalysis } from '../../../../common/analytics'; @@ -401,13 +400,11 @@ export const useNavigateToWizardWithClonedJob = () => { const { services: { notifications: { toasts }, - savedObjects, + data: { dataViews }, }, } = useMlKibana(); const navigateToPath = useNavigateToPath(); - const savedObjectsClient = savedObjects.client; - return async (item: Pick) => { const sourceIndex = Array.isArray(item.config.source.index) ? item.config.source.index.join(',') @@ -415,19 +412,9 @@ export const useNavigateToWizardWithClonedJob = () => { let sourceIndexId; try { - const response = await savedObjectsClient.find({ - type: 'index-pattern', - perPage: 10, - search: `"${sourceIndex}"`, - searchFields: ['title'], - fields: ['title'], - }); - - const ip = response.savedObjects.find( - (obj) => obj.attributes.title.toLowerCase() === sourceIndex.toLowerCase() - ); - if (ip !== undefined) { - sourceIndexId = ip.id; + const dv = (await dataViews.find(sourceIndex)).find(({ title }) => title === sourceIndex); + if (dv !== undefined) { + sourceIndexId = dv.id; } else { toasts.addDanger( i18n.translate('xpack.ml.dataframe.analyticsList.noSourceIndexPatternForClone', { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_delete/delete_action_name.test.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_delete/delete_action_name.test.tsx index 6b26e3823d2ef..ad6a59bf01c0e 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_delete/delete_action_name.test.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_delete/delete_action_name.test.tsx @@ -30,7 +30,7 @@ jest.mock('../../../../../../application/util/dependency_cache', () => ({ jest.mock('../../../../../contexts/kibana', () => ({ useMlKibana: () => ({ - services: mockCoreServices.createStart(), + services: { ...mockCoreServices.createStart(), data: { data_view: { find: jest.fn() } } }, }), useNotifications: () => { return { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_delete/use_delete_action.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_delete/use_delete_action.tsx index 91871015d2add..0d2025c0d049a 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_delete/use_delete_action.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_delete/use_delete_action.tsx @@ -8,9 +8,6 @@ import React, { useEffect, useMemo, useState } from 'react'; import { i18n } from '@kbn/i18n'; - -import { IIndexPattern } from 'src/plugins/data/common'; - import { extractErrorMessage } from '../../../../../../../common/util/errors'; import { useMlKibana } from '../../../../../contexts/kibana'; @@ -48,8 +45,9 @@ export const useDeleteAction = (canDeleteDataFrameAnalytics: boolean) => { const [indexPatternExists, setIndexPatternExists] = useState(false); const [isLoading, setIsLoading] = useState(false); - const { savedObjects } = useMlKibana().services; - const savedObjectsClient = savedObjects.client; + const { + data: { dataViews }, + } = useMlKibana().services; const indexName = item?.config.dest.index ?? ''; @@ -57,17 +55,8 @@ export const useDeleteAction = (canDeleteDataFrameAnalytics: boolean) => { const checkIndexPatternExists = async () => { try { - const response = await savedObjectsClient.find({ - type: 'index-pattern', - perPage: 10, - search: `"${indexName}"`, - searchFields: ['title'], - fields: ['title'], - }); - const ip = response.savedObjects.find( - (obj) => obj.attributes.title.toLowerCase() === indexName.toLowerCase() - ); - if (ip !== undefined) { + const dv = (await dataViews.find(indexName)).find(({ title }) => title === indexName); + if (dv !== undefined) { setIndexPatternExists(true); } else { setIndexPatternExists(false); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/source_selection/source_selection.test.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/source_selection/source_selection.test.tsx index 7e90a4e3ed44a..6e663318d2dc6 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/source_selection/source_selection.test.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/source_selection/source_selection.test.tsx @@ -87,6 +87,7 @@ jest.mock('../../../../../util/index_utils', () => { async (id: string): Promise => { return { indexPattern: { + // @ts-expect-error fields should not be empty fields: [], title: id === 'the-remote-saved-search-id' diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts index a5fabc12c83df..1dccd54f68a38 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts @@ -7,7 +7,7 @@ import { CoreSetup } from 'src/core/public'; -import { IndexPattern } from '../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../src/plugins/data_views/public'; import { SavedSearchQuery } from '../../../contexts/ml'; import { OMIT_FIELDS } from '../../../../../common/constants/field_types'; @@ -22,15 +22,12 @@ import { RuntimeMappings } from '../../../../../common/types/fields'; const MAX_EXAMPLES_DEFAULT: number = 10; export class DataLoader { - private _indexPattern: IndexPattern; + private _indexPattern: DataView; private _runtimeMappings: RuntimeMappings; private _indexPatternTitle: IndexPatternTitle = ''; private _maxExamples: number = MAX_EXAMPLES_DEFAULT; - constructor( - indexPattern: IndexPattern, - toastNotifications?: CoreSetup['notifications']['toasts'] - ) { + constructor(indexPattern: DataView, toastNotifications?: CoreSetup['notifications']['toasts']) { this._indexPattern = indexPattern; this._runtimeMappings = this._indexPattern.getComputedFields().runtimeFields as RuntimeMappings; this._indexPatternTitle = indexPattern.title; diff --git a/x-pack/plugins/ml/public/application/explorer/components/explorer_query_bar/explorer_query_bar.tsx b/x-pack/plugins/ml/public/application/explorer/components/explorer_query_bar/explorer_query_bar.tsx index 6e8b5f762558f..f57d2c1b01d98 100644 --- a/x-pack/plugins/ml/public/application/explorer/components/explorer_query_bar/explorer_query_bar.tsx +++ b/x-pack/plugins/ml/public/application/explorer/components/explorer_query_bar/explorer_query_bar.tsx @@ -10,7 +10,7 @@ import { EuiCode, EuiInputPopover } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { fromKueryExpression, luceneStringToDsl, toElasticsearchQuery } from '@kbn/es-query'; import { Query, QueryStringInput } from '../../../../../../../../src/plugins/data/public'; -import { IIndexPattern } from '../../../../../../../../src/plugins/data/common'; +import { DataView } from '../../../../../../../../src/plugins/data_views/common'; import { SEARCH_QUERY_LANGUAGE, ErrorMessage } from '../../../../../common/constants/search'; import { explorerService } from '../../explorer_dashboard_service'; import { InfluencersFilterQuery } from '../../../../../common/types/es_client'; @@ -24,7 +24,7 @@ export function getKqlQueryValues({ }: { inputString: string | { [key: string]: any }; queryLanguage: string; - indexPattern: IIndexPattern; + indexPattern: DataView; }): { clearSettings: boolean; settings: any } { let influencersFilterQuery: InfluencersFilterQuery = {}; const filteredFields: string[] = []; @@ -89,7 +89,7 @@ function getInitSearchInputState({ interface ExplorerQueryBarProps { filterActive: boolean; filterPlaceHolder: string; - indexPattern: IIndexPattern; + indexPattern: DataView; queryString?: string; updateLanguage: (language: string) => void; } diff --git a/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/editor.test.tsx b/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/editor.test.tsx index f6769abb610b8..11e4c14cd4ab2 100644 --- a/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/editor.test.tsx +++ b/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/editor.test.tsx @@ -16,7 +16,7 @@ import React from 'react'; import { CustomUrlEditor } from './editor'; import { TIME_RANGE_TYPE, URL_TYPE } from './constants'; import { CustomUrlSettings } from './utils'; -import { IIndexPattern } from '../../../../../../../../src/plugins/data/common'; +import { DataView } from '../../../../../../../../src/plugins/data_views/common'; function prepareTest(customUrl: CustomUrlSettings, setEditCustomUrlFn: (url: UrlConfig) => void) { const savedCustomUrls = [ @@ -50,7 +50,7 @@ function prepareTest(customUrl: CustomUrlSettings, setEditCustomUrlFn: (url: Url const indexPatterns = [ { id: 'pattern1', title: 'Index Pattern 1' }, { id: 'pattern2', title: 'Index Pattern 2' }, - ] as IIndexPattern[]; + ] as DataView[]; const queryEntityFieldNames = ['airline']; diff --git a/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/editor.tsx b/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/editor.tsx index e22eb1484df2e..7dd779ead7892 100644 --- a/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/editor.tsx +++ b/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/editor.tsx @@ -29,7 +29,7 @@ import { isValidLabel } from '../../../util/custom_url_utils'; import { TIME_RANGE_TYPE, URL_TYPE } from './constants'; import { UrlConfig } from '../../../../../common/types/custom_urls'; -import { IIndexPattern } from '../../../../../../../../src/plugins/data/common'; +import { DataView } from '../../../../../../../../src/plugins/data_views/common'; function getLinkToOptions() { return [ @@ -59,7 +59,7 @@ interface CustomUrlEditorProps { setEditCustomUrl: (url: any) => void; savedCustomUrls: UrlConfig[]; dashboards: any[]; - indexPatterns: IIndexPattern[]; + indexPatterns: DataView[]; queryEntityFieldNames: string[]; } diff --git a/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/utils.d.ts b/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/utils.d.ts index 87000cdabd913..1f815759c6242 100644 --- a/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/utils.d.ts +++ b/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/utils.d.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IIndexPattern } from 'src/plugins/data/common'; +import { DataView } from '../../../../../../../../src/plugins/data_views/common'; import { UrlConfig } from '../../../../../common/types/custom_urls'; import { Job } from '../../../../../common/types/anomaly_detection_jobs'; import { TimeRangeType } from './constants'; @@ -34,7 +34,7 @@ export function isValidCustomUrlSettingsTimeRange(timeRangeSettings: any): boole export function getNewCustomUrlDefaults( job: Job, dashboards: any[], - indexPatterns: IIndexPattern[] + indexPatterns: DataView[] ): CustomUrlSettings; export function getQueryEntityFieldNames(job: Job): string[]; export function isValidCustomUrlSettings( diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/edit_utils.d.ts b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/edit_utils.d.ts index 3a14715bca4b9..32e99e3e433e0 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/edit_utils.d.ts +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/edit_utils.d.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IIndexPattern } from 'src/plugins/data/common'; +import type { DataView } from 'src/plugins/data_views/common'; export function loadSavedDashboards(maxNumber: number): Promise; -export function loadIndexPatterns(maxNumber: number): Promise; +export function loadIndexPatterns(maxNumber: number): Promise; diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/edit_utils.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/edit_utils.js index 8ce92ffa38479..ad192a738174e 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/edit_utils.js +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/edit_utils.js @@ -8,7 +8,7 @@ import { difference } from 'lodash'; import { getNewJobLimits } from '../../../../services/ml_server_info'; import { processCreatedBy } from '../../../../../../common/util/job_utils'; -import { getSavedObjectsClient } from '../../../../util/dependency_cache'; +import { getSavedObjectsClient, getDataViews } from '../../../../util/dependency_cache'; import { ml } from '../../../../services/ml_api_service'; export function saveJob(job, newJobData, finish) { @@ -107,26 +107,12 @@ export function loadIndexPatterns(maxNumber) { // TODO - amend loadIndexPatterns in index_utils.js to do the request, // without needing an Angular Provider. return new Promise((resolve, reject) => { - const savedObjectsClient = getSavedObjectsClient(); - savedObjectsClient - .find({ - type: 'index-pattern', - fields: ['title'], - perPage: maxNumber, - }) - .then((resp) => { - const savedObjects = resp.savedObjects; - if (savedObjects !== undefined) { - const indexPatterns = savedObjects.map((savedObj) => { - return { id: savedObj.id, title: savedObj.attributes.title }; - }); - - indexPatterns.sort((dash1, dash2) => { - return dash1.title.localeCompare(dash2.title); - }); - - resolve(indexPatterns); - } + const dataViewsContract = getDataViews(); + dataViewsContract + .find('*', maxNumber) + .then((dataViews) => { + const sortedDataViews = dataViews.sort((a, b) => a.title.localeCompare(b.title)); + resolve(sortedDataViews); }) .catch((resp) => { reject(resp); diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/tabs/custom_urls.tsx b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/tabs/custom_urls.tsx index ce93080558016..46ac1dbd01b7f 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/tabs/custom_urls.tsx +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/tabs/custom_urls.tsx @@ -37,7 +37,7 @@ import { loadSavedDashboards, loadIndexPatterns } from '../edit_utils'; import { openCustomUrlWindow } from '../../../../../util/custom_url_utils'; import { Job } from '../../../../../../../common/types/anomaly_detection_jobs'; import { UrlConfig } from '../../../../../../../common/types/custom_urls'; -import { IIndexPattern } from '../../../../../../../../../../src/plugins/data/common'; +import { DataView } from '../../../../../../../../../../src/plugins/data_views/common'; import { MlKibanaReactContextValue } from '../../../../../contexts/kibana'; const MAX_NUMBER_DASHBOARDS = 1000; @@ -54,7 +54,7 @@ interface CustomUrlsProps { interface CustomUrlsState { customUrls: UrlConfig[]; dashboards: any[]; - indexPatterns: IIndexPattern[]; + indexPatterns: DataView[]; queryEntityFieldNames: string[]; editorOpen: boolean; editorSettings?: CustomUrlSettings; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/chart_loader.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/chart_loader.ts index 9c8f34260def0..5898a9dec1ad3 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/chart_loader.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/chart_loader.ts @@ -19,7 +19,7 @@ import { ml } from '../../../../services/ml_api_service'; import { mlResultsService } from '../../../../services/results_service'; import { getCategoryFields as getCategoryFieldsOrig } from './searches'; import { aggFieldPairsCanBeCharted } from '../job_creator/util/general'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/common'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/common'; type DetectorIndex = number; export interface LineChartPoint { @@ -41,7 +41,7 @@ export class ChartLoader { private _timeFieldName: string = ''; private _query: object = {}; - constructor(indexPattern: IndexPattern, query: object) { + constructor(indexPattern: DataView, query: object) { this._indexPatternTitle = indexPattern.title; this._query = query; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/index_pattern_context.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/index_pattern_context.ts index cb842937f1ede..9667465eb210d 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/index_pattern_context.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/index_pattern_context.ts @@ -7,7 +7,7 @@ import React from 'react'; -import { IIndexPattern } from 'src/plugins/data/public'; +import { DataView } from '../../../../../../../../src/plugins/data_views/common'; -export type IndexPatternContextValue = IIndexPattern | null; +export type IndexPatternContextValue = DataView | null; export const IndexPatternContext = React.createContext(null); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts index 35847839b02a0..3d8c34e0e5967 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts @@ -21,7 +21,7 @@ import { JOB_TYPE } from '../../../../../../common/constants/new_job'; import { getRichDetectors } from './util/general'; import { isValidJson } from '../../../../../../common/util/validation_utils'; import { ml } from '../../../../services/ml_api_service'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; export interface RichDetector { agg: Aggregation | null; @@ -40,11 +40,7 @@ export class AdvancedJobCreator extends JobCreator { private _richDetectors: RichDetector[] = []; private _queryString: string; - constructor( - indexPattern: IndexPattern, - savedSearch: SavedSearchSavedObject | null, - query: object - ) { + constructor(indexPattern: DataView, savedSearch: SavedSearchSavedObject | null, query: object) { super(indexPattern, savedSearch, query); this._queryString = JSON.stringify(this._datafeed_config.query); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index 128a541ff9f96..b46d3b539b44a 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -6,7 +6,7 @@ */ import { isEqual } from 'lodash'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; import { SavedSearchSavedObject } from '../../../../../../common/types/kibana'; import { JobCreator } from './job_creator'; import { Field, Aggregation, mlCategory } from '../../../../../../common/types/fields'; @@ -47,11 +47,7 @@ export class CategorizationJobCreator extends JobCreator { private _partitionFieldName: string | null = null; private _ccsVersionFailure: boolean = false; - constructor( - indexPattern: IndexPattern, - savedSearch: SavedSearchSavedObject | null, - query: object - ) { + constructor(indexPattern: DataView, savedSearch: SavedSearchSavedObject | null, query: object) { super(indexPattern, savedSearch, query); this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; this._examplesLoader = new CategorizationExamplesLoader(this, indexPattern, query); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts index e6cfe52933617..a44b4bdef60c4 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts @@ -40,13 +40,13 @@ import { filterRuntimeMappings } from './util/filter_runtime_mappings'; import { parseInterval } from '../../../../../../common/util/parse_interval'; import { Calendar } from '../../../../../../common/types/calendars'; import { mlCalendarService } from '../../../../services/calendar_service'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; import { getDatafeedAggregations } from '../../../../../../common/util/datafeed_utils'; import { getFirstKeyInObject } from '../../../../../../common/util/object_utils'; export class JobCreator { protected _type: JOB_TYPE = JOB_TYPE.SINGLE_METRIC; - protected _indexPattern: IndexPattern; + protected _indexPattern: DataView; protected _savedSearch: SavedSearchSavedObject | null; protected _indexPatternTitle: IndexPatternTitle = ''; protected _job_config: Job; @@ -74,11 +74,7 @@ export class JobCreator { protected _wizardInitialized$ = new BehaviorSubject(false); public wizardInitialized$ = this._wizardInitialized$.asObservable(); - constructor( - indexPattern: IndexPattern, - savedSearch: SavedSearchSavedObject | null, - query: object - ) { + constructor(indexPattern: DataView, savedSearch: SavedSearchSavedObject | null, query: object) { this._indexPattern = indexPattern; this._savedSearch = savedSearch; this._indexPatternTitle = indexPattern.title; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts index 8c77ae5def102..6af3df888514c 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts @@ -10,7 +10,7 @@ import { SingleMetricJobCreator } from './single_metric_job_creator'; import { MultiMetricJobCreator } from './multi_metric_job_creator'; import { PopulationJobCreator } from './population_job_creator'; import { AdvancedJobCreator } from './advanced_job_creator'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; import { CategorizationJobCreator } from './categorization_job_creator'; import { RareJobCreator } from './rare_job_creator'; @@ -18,7 +18,7 @@ import { JOB_TYPE } from '../../../../../../common/constants/new_job'; export const jobCreatorFactory = (jobType: JOB_TYPE) => - (indexPattern: IndexPattern, savedSearch: SavedSearchSavedObject | null, query: object) => { + (indexPattern: DataView, savedSearch: SavedSearchSavedObject | null, query: object) => { let jc; switch (jobType) { case JOB_TYPE.SINGLE_METRIC: diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts index f63aa1b569a2c..12543f34003d5 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts @@ -17,7 +17,7 @@ import { Job, Datafeed, Detector } from '../../../../../../common/types/anomaly_ import { createBasicDetector } from './util/default_configs'; import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/new_job'; import { getRichDetectors } from './util/general'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; import { isSparseDataJob } from './util/general'; export class MultiMetricJobCreator extends JobCreator { @@ -27,11 +27,7 @@ export class MultiMetricJobCreator extends JobCreator { protected _type: JOB_TYPE = JOB_TYPE.MULTI_METRIC; - constructor( - indexPattern: IndexPattern, - savedSearch: SavedSearchSavedObject | null, - query: object - ) { + constructor(indexPattern: DataView, savedSearch: SavedSearchSavedObject | null, query: object) { super(indexPattern, savedSearch, query); this.createdBy = CREATED_BY_LABEL.MULTI_METRIC; this._wizardInitialized$.next(true); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts index 24b3192231211..7f001ce334462 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts @@ -17,7 +17,7 @@ import { Job, Datafeed, Detector } from '../../../../../../common/types/anomaly_ import { createBasicDetector } from './util/default_configs'; import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/new_job'; import { getRichDetectors } from './util/general'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; export class PopulationJobCreator extends JobCreator { // a population job has one overall over (split) field, which is the same for all detectors @@ -26,11 +26,7 @@ export class PopulationJobCreator extends JobCreator { private _byFields: SplitField[] = []; protected _type: JOB_TYPE = JOB_TYPE.POPULATION; - constructor( - indexPattern: IndexPattern, - savedSearch: SavedSearchSavedObject | null, - query: object - ) { + constructor(indexPattern: DataView, savedSearch: SavedSearchSavedObject | null, query: object) { super(indexPattern, savedSearch, query); this.createdBy = CREATED_BY_LABEL.POPULATION; this._wizardInitialized$.next(true); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/rare_job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/rare_job_creator.ts index 73050dc4b7834..8973aa655b83d 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/rare_job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/rare_job_creator.ts @@ -11,7 +11,7 @@ import { Field, SplitField, Aggregation } from '../../../../../../common/types/f import { Job, Datafeed, Detector } from '../../../../../../common/types/anomaly_detection_jobs'; import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/new_job'; import { getRichDetectors } from './util/general'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; import { isSparseDataJob } from './util/general'; import { ML_JOB_AGGREGATION } from '../../../../../../common/constants/aggregation_types'; @@ -26,11 +26,7 @@ export class RareJobCreator extends JobCreator { private _rareAgg: Aggregation; private _freqRareAgg: Aggregation; - constructor( - indexPattern: IndexPattern, - savedSearch: SavedSearchSavedObject | null, - query: object - ) { + constructor(indexPattern: DataView, savedSearch: SavedSearchSavedObject | null, query: object) { super(indexPattern, savedSearch, query); this.createdBy = CREATED_BY_LABEL.RARE; this._wizardInitialized$.next(true); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts index 57ff76979ea14..9c4fd52888c82 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts @@ -22,17 +22,13 @@ import { } from '../../../../../../common/constants/aggregation_types'; import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/new_job'; import { getRichDetectors } from './util/general'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; import { isSparseDataJob } from './util/general'; export class SingleMetricJobCreator extends JobCreator { protected _type: JOB_TYPE = JOB_TYPE.SINGLE_METRIC; - constructor( - indexPattern: IndexPattern, - savedSearch: SavedSearchSavedObject | null, - query: object - ) { + constructor(indexPattern: DataView, savedSearch: SavedSearchSavedObject | null, query: object) { super(indexPattern, savedSearch, query); this.createdBy = CREATED_BY_LABEL.SINGLE_METRIC; this._wizardInitialized$.next(true); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts index 641eda3dbf3e8..6e65bde879379 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../../../src/plugins/data_views/public'; import { IndexPatternTitle } from '../../../../../../common/types/kibana'; import { CategorizationJobCreator } from '../job_creator'; import { ml } from '../../../../services/ml_api_service'; @@ -20,7 +20,7 @@ export class CategorizationExamplesLoader { private _timeFieldName: string = ''; private _query: object = {}; - constructor(jobCreator: CategorizationJobCreator, indexPattern: IndexPattern, query: object) { + constructor(jobCreator: CategorizationJobCreator, indexPattern: DataView, query: object) { this._jobCreator = jobCreator; this._indexPatternTitle = indexPattern.title; this._query = query; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts b/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts index ba750795e4f8f..03428bd47e490 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts @@ -5,15 +5,15 @@ * 2.0. */ -import { ApplicationStart } from 'kibana/public'; -import { IndexPatternsContract } from '../../../../../../../../../src/plugins/data/public'; +import type { ApplicationStart } from 'kibana/public'; +import type { DataViewsContract } from '../../../../../../../../../src/plugins/data_views/public'; import { mlJobService } from '../../../../services/job_service'; import { loadIndexPatterns, getIndexPatternIdFromName } from '../../../../util/index_utils'; import { Datafeed, Job } from '../../../../../../common/types/anomaly_detection_jobs'; import { CREATED_BY_LABEL, JOB_TYPE } from '../../../../../../common/constants/new_job'; export async function preConfiguredJobRedirect( - indexPatterns: IndexPatternsContract, + indexPatterns: DataViewsContract, basePath: string, navigateToUrl: ApplicationStart['navigateToUrl'] ) { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.test.ts b/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.test.ts index 3f19f3137934e..12beda414bbea 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.test.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.test.ts @@ -6,7 +6,7 @@ */ import { IUiSettingsClient } from 'kibana/public'; -import { IIndexPattern } from '../../../../../../../../src/plugins/data/common'; +import { DataView } from '../../../../../../../../src/plugins/data_views/common'; import { SavedSearchSavedObject } from '../../../../../common/types/kibana'; import { createSearchItems } from './new_job_utils'; @@ -14,7 +14,7 @@ describe('createSearchItems', () => { const kibanaConfig = {} as IUiSettingsClient; const indexPattern = { fields: [], - } as unknown as IIndexPattern; + } as unknown as DataView; let savedSearch = {} as unknown as SavedSearchSavedObject; beforeEach(() => { diff --git a/x-pack/plugins/ml/public/application/routing/resolvers.ts b/x-pack/plugins/ml/public/application/routing/resolvers.ts index f6364ecc6568f..3479005809efb 100644 --- a/x-pack/plugins/ml/public/application/routing/resolvers.ts +++ b/x-pack/plugins/ml/public/application/routing/resolvers.ts @@ -11,7 +11,7 @@ import { checkGetJobsCapabilitiesResolver } from '../capabilities/check_capabili import { getMlNodeCount } from '../ml_nodes_check/check_ml_nodes'; import { loadMlServerInfo } from '../services/ml_server_info'; -import { IndexPatternsContract } from '../../../../../../src/plugins/data/public'; +import type { DataViewsContract } from '../../../../../../src/plugins/data_views/public'; export interface Resolvers { [name: string]: () => Promise; @@ -21,7 +21,7 @@ export interface ResolverResults { } interface BasicResolverDependencies { - indexPatterns: IndexPatternsContract; + indexPatterns: DataViewsContract; redirectToMlAccessDeniedPage: () => Promise; } diff --git a/x-pack/plugins/ml/public/application/routing/router.tsx b/x-pack/plugins/ml/public/application/routing/router.tsx index c2129ef18df3a..847dcc1ae1107 100644 --- a/x-pack/plugins/ml/public/application/routing/router.tsx +++ b/x-pack/plugins/ml/public/application/routing/router.tsx @@ -9,9 +9,13 @@ import React, { useEffect, FC } from 'react'; import { useHistory, useLocation, Router, Route, RouteProps } from 'react-router-dom'; import { Location } from 'history'; -import { AppMountParameters, IUiSettingsClient, ChromeStart } from 'kibana/public'; -import { ChromeBreadcrumb } from 'kibana/public'; -import { IndexPatternsContract } from 'src/plugins/data/public'; +import type { + AppMountParameters, + IUiSettingsClient, + ChromeStart, + ChromeBreadcrumb, +} from 'kibana/public'; +import type { DataViewsContract } from 'src/plugins/data_views/public'; import { useMlKibana, useNavigateToPath } from '../contexts/kibana'; import { MlContext, MlContextValue } from '../contexts/ml'; @@ -39,7 +43,7 @@ export interface PageProps { interface PageDependencies { config: IUiSettingsClient; history: AppMountParameters['history']; - indexPatterns: IndexPatternsContract; + indexPatterns: DataViewsContract; setBreadcrumbs: ChromeStart['setBreadcrumbs']; redirectToMlAccessDeniedPage: () => Promise; } diff --git a/x-pack/plugins/ml/public/application/services/field_format_service.ts b/x-pack/plugins/ml/public/application/services/field_format_service.ts index 18b489682318e..fe6fc7751bb85 100644 --- a/x-pack/plugins/ml/public/application/services/field_format_service.ts +++ b/x-pack/plugins/ml/public/application/services/field_format_service.ts @@ -8,7 +8,7 @@ import { mlFunctionToESAggregation } from '../../../common/util/job_utils'; import { getIndexPatternById, getIndexPatternIdFromName } from '../util/index_utils'; import { mlJobService } from './job_service'; -import { IndexPattern } from '../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../src/plugins/data_views/public'; type FormatsByJobId = Record; type IndexPatternIdsByJob = Record; @@ -66,11 +66,7 @@ class FieldFormatService { // Utility for returning the FieldFormat from a full populated Kibana index pattern object // containing the list of fields by name with their formats. - getFieldFormatFromIndexPattern( - fullIndexPattern: IndexPattern, - fieldName: string, - esAggName: string - ) { + getFieldFormatFromIndexPattern(fullIndexPattern: DataView, fieldName: string, esAggName: string) { // Don't use the field formatter for distinct count detectors as // e.g. distinct_count(clientip) should be formatted as a count, not as an IP address. let fieldFormat; diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/load_new_job_capabilities.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/load_new_job_capabilities.ts index d3b407c2bb65a..02b2e573f8c69 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities/load_new_job_capabilities.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/load_new_job_capabilities.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IIndexPattern, IndexPatternsContract } from '../../../../../../../src/plugins/data/public'; +import { DataView, DataViewsContract } from '../../../../../../../src/plugins/data_views/public'; import { getIndexPatternAndSavedSearch } from '../../util/index_utils'; import { JobType } from '../../../../common/types/saved_objects'; import { newJobCapsServiceAnalytics } from '../new_job_capabilities/new_job_capabilities_service_analytics'; @@ -19,7 +19,7 @@ export const DATA_FRAME_ANALYTICS = 'data-frame-analytics'; export function loadNewJobCapabilities( indexPatternId: string, savedSearchId: string, - indexPatterns: IndexPatternsContract, + indexPatterns: DataViewsContract, jobType: JobType ) { return new Promise(async (resolve, reject) => { @@ -29,7 +29,7 @@ export function loadNewJobCapabilities( if (indexPatternId !== undefined) { // index pattern is being used - const indexPattern: IIndexPattern = await indexPatterns.get(indexPatternId); + const indexPattern: DataView = await indexPatterns.get(indexPatternId); await serviceToUse.initializeFromIndexPattern(indexPattern); resolve(serviceToUse.newJobCaps); } else if (savedSearchId !== undefined) { diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities._service.test.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities._service.test.ts index 8c515255927b4..49c8b08007d52 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities._service.test.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities._service.test.ts @@ -6,7 +6,7 @@ */ import { newJobCapsService } from './new_job_capabilities_service'; -import { IndexPattern } from '../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../src/plugins/data_views/public'; // there is magic happening here. starting the include name with `mock..` // ensures it can be lazily loaded by the jest.mock function below. @@ -23,7 +23,7 @@ jest.mock('../ml_api_service', () => ({ const indexPattern = { id: 'cloudwatch-*', title: 'cloudwatch-*', -} as unknown as IndexPattern; +} as unknown as DataView; describe('new_job_capabilities_service', () => { describe('cloudwatch newJobCaps()', () => { diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts index c17f379355cea..45dc71ed6a6b9 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts @@ -11,7 +11,8 @@ import { FieldId, EVENT_RATE_FIELD_ID, } from '../../../../common/types/fields'; -import { ES_FIELD_TYPES, IIndexPattern } from '../../../../../../../src/plugins/data/public'; +import { ES_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; +import { DataView } from '../../../../../../../src/plugins/data_views/public'; import { ml } from '../ml_api_service'; import { processTextAndKeywordFields, NewJobCapabilitiesServiceBase } from './new_job_capabilities'; @@ -36,7 +37,7 @@ class NewJobCapsService extends NewJobCapabilitiesServiceBase { } public async initializeFromIndexPattern( - indexPattern: IIndexPattern, + indexPattern: DataView, includeEventRateField = true, removeTextFields = true ) { diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts index 3a362a88e40bb..f8f9ae6b2b0a3 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts @@ -6,7 +6,8 @@ */ import { Field, NewJobCapsResponse } from '../../../../common/types/fields'; -import { ES_FIELD_TYPES, IIndexPattern } from '../../../../../../../src/plugins/data/public'; +import { ES_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; +import { DataView } from '../../../../../../../src/plugins/data_views/public'; import { processTextAndKeywordFields, NewJobCapabilitiesServiceBase } from './new_job_capabilities'; import { ml } from '../ml_api_service'; @@ -42,7 +43,7 @@ export function removeNestedFieldChildren(resp: NewJobCapsResponse, indexPattern } class NewJobCapsServiceAnalytics extends NewJobCapabilitiesServiceBase { - public async initializeFromIndexPattern(indexPattern: IIndexPattern) { + public async initializeFromIndexPattern(indexPattern: DataView) { try { const resp: NewJobCapsResponse = await ml.dataFrameAnalytics.newJobCapsAnalytics( indexPattern.title, diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/remove_nested_field_children.test.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/remove_nested_field_children.test.ts index eaf30d9894f60..ec2a2a1077a7f 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities/remove_nested_field_children.test.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/remove_nested_field_children.test.ts @@ -6,7 +6,7 @@ */ import { removeNestedFieldChildren } from './new_job_capabilities_service_analytics'; -import { IndexPattern } from '../../../../../../../src/plugins/data/public'; +import type { DataView } from '../../../../../../../src/plugins/data_views/public'; // there is magic happening here. starting the include name with `mock..` // ensures it can be lazily loaded by the jest.mock function below. @@ -15,7 +15,7 @@ import nestedFieldIndexResponse from '../__mocks__/nested_field_index_response.j const indexPattern = { id: 'nested-field-index', title: 'nested-field-index', -} as unknown as IndexPattern; +} as unknown as DataView; describe('removeNestedFieldChildren', () => { describe('cloudwatch newJobCapsAnalytics()', () => { diff --git a/x-pack/plugins/ml/public/application/util/dependency_cache.ts b/x-pack/plugins/ml/public/application/util/dependency_cache.ts index 4a3194ed4113f..7b6b75677dddd 100644 --- a/x-pack/plugins/ml/public/application/util/dependency_cache.ts +++ b/x-pack/plugins/ml/public/application/util/dependency_cache.ts @@ -19,8 +19,9 @@ import type { ChromeRecentlyAccessed, IBasePath, } from 'kibana/public'; -import type { IndexPatternsContract, DataPublicPluginStart } from 'src/plugins/data/public'; +import type { DataPublicPluginStart } from 'src/plugins/data/public'; import type { SharePluginStart } from 'src/plugins/share/public'; +import type { DataViewsContract } from '../../../../../../src/plugins/data_views/public'; import type { SecurityPluginSetup } from '../../../../security/public'; import type { MapsStartApi } from '../../../../maps/public'; import type { DataVisualizerPluginStart } from '../../../../data_visualizer/public'; @@ -28,7 +29,7 @@ import type { DataVisualizerPluginStart } from '../../../../data_visualizer/publ export interface DependencyCache { timefilter: DataPublicPluginSetup['query']['timefilter'] | null; config: IUiSettingsClient | null; - indexPatterns: IndexPatternsContract | null; + indexPatterns: DataViewsContract | null; chrome: ChromeStart | null; docLinks: DocLinksStart | null; toastNotifications: ToastsStart | null; @@ -45,6 +46,7 @@ export interface DependencyCache { urlGenerators: SharePluginStart['urlGenerators'] | null; maps: MapsStartApi | null; dataVisualizer: DataVisualizerPluginStart | null; + dataViews: DataViewsContract | null; } const cache: DependencyCache = { @@ -67,6 +69,7 @@ const cache: DependencyCache = { urlGenerators: null, maps: null, dataVisualizer: null, + dataViews: null, }; export function setDependencyCache(deps: Partial) { @@ -88,6 +91,7 @@ export function setDependencyCache(deps: Partial) { cache.i18n = deps.i18n || null; cache.urlGenerators = deps.urlGenerators || null; cache.dataVisualizer = deps.dataVisualizer || null; + cache.dataViews = deps.dataViews || null; } export function getTimefilter() { @@ -208,6 +212,13 @@ export function getGetUrlGenerator() { return cache.urlGenerators.getUrlGenerator; } +export function getDataViews() { + if (cache.dataViews === null) { + throw new Error("dataViews hasn't been initialized"); + } + return cache.dataViews; +} + export function clearCache() { Object.keys(cache).forEach((k) => { cache[k as keyof DependencyCache] = null; diff --git a/x-pack/plugins/ml/public/application/util/field_types_utils.test.ts b/x-pack/plugins/ml/public/application/util/field_types_utils.test.ts index 440ac411e8ee7..0c50dc9efa343 100644 --- a/x-pack/plugins/ml/public/application/util/field_types_utils.test.ts +++ b/x-pack/plugins/ml/public/application/util/field_types_utils.test.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { IFieldType, KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; +import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; +import { DataViewField } from '../../../../../../src/plugins/data_views/common'; import { ML_JOB_FIELD_TYPES } from '../../../common/constants/field_types'; import { kbnTypeToMLJobType, @@ -16,28 +17,34 @@ import { describe('ML - field type utils', () => { describe('kbnTypeToMLJobType', () => { test('returns correct ML_JOB_FIELD_TYPES for KBN_FIELD_TYPES', () => { - const field: IFieldType = { + // @ts-ignore reassigning missing properties + const field: DataViewField = { type: KBN_FIELD_TYPES.NUMBER, name: KBN_FIELD_TYPES.NUMBER, aggregatable: true, }; expect(kbnTypeToMLJobType(field)).toBe(ML_JOB_FIELD_TYPES.NUMBER); + // @ts-ignore reassigning read-only type field.type = KBN_FIELD_TYPES.DATE; expect(kbnTypeToMLJobType(field)).toBe(ML_JOB_FIELD_TYPES.DATE); + // @ts-ignore reassigning read-only type field.type = KBN_FIELD_TYPES.IP; expect(kbnTypeToMLJobType(field)).toBe(ML_JOB_FIELD_TYPES.IP); + // @ts-ignore reassigning read-only type field.type = KBN_FIELD_TYPES.BOOLEAN; expect(kbnTypeToMLJobType(field)).toBe(ML_JOB_FIELD_TYPES.BOOLEAN); + // @ts-ignore reassigning read-only type field.type = KBN_FIELD_TYPES.GEO_POINT; expect(kbnTypeToMLJobType(field)).toBe(ML_JOB_FIELD_TYPES.GEO_POINT); }); test('returns ML_JOB_FIELD_TYPES.KEYWORD for aggregatable KBN_FIELD_TYPES.STRING', () => { - const field: IFieldType = { + // @ts-ignore reassigning missing properties + const field: DataViewField = { type: KBN_FIELD_TYPES.STRING, name: KBN_FIELD_TYPES.STRING, aggregatable: true, @@ -46,7 +53,8 @@ describe('ML - field type utils', () => { }); test('returns ML_JOB_FIELD_TYPES.TEXT for non-aggregatable KBN_FIELD_TYPES.STRING', () => { - const field: IFieldType = { + // @ts-ignore reassigning missing properties + const field: DataViewField = { type: KBN_FIELD_TYPES.STRING, name: KBN_FIELD_TYPES.STRING, aggregatable: false, @@ -55,7 +63,8 @@ describe('ML - field type utils', () => { }); test('returns undefined for non-aggregatable "foo"', () => { - const field: IFieldType = { + // @ts-ignore reassigning missing properties + const field: DataViewField = { type: 'foo', name: 'foo', aggregatable: false, diff --git a/x-pack/plugins/ml/public/application/util/field_types_utils.ts b/x-pack/plugins/ml/public/application/util/field_types_utils.ts index 0cb21fec1862f..c02a1cbec56ec 100644 --- a/x-pack/plugins/ml/public/application/util/field_types_utils.ts +++ b/x-pack/plugins/ml/public/application/util/field_types_utils.ts @@ -8,12 +8,13 @@ import { i18n } from '@kbn/i18n'; import { ML_JOB_FIELD_TYPES } from '../../../common/constants/field_types'; -import { IFieldType, KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; +import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; +import { DataViewField } from '../../../../../../src/plugins/data_views/common'; // convert kibana types to ML Job types // this is needed because kibana types only have string and not text and keyword. // and we can't use ES_FIELD_TYPES because it has no NUMBER type -export function kbnTypeToMLJobType(field: IFieldType) { +export function kbnTypeToMLJobType(field: DataViewField) { // Return undefined if not one of the supported data visualizer field types. let type; switch (field.type) { diff --git a/x-pack/plugins/ml/public/application/util/index_utils.ts b/x-pack/plugins/ml/public/application/util/index_utils.ts index 9d705c8cd725f..b4f46d4df0cbb 100644 --- a/x-pack/plugins/ml/public/application/util/index_utils.ts +++ b/x-pack/plugins/ml/public/application/util/index_utils.ts @@ -6,33 +6,20 @@ */ import { i18n } from '@kbn/i18n'; -import { - IndexPattern, - IIndexPattern, - IndexPatternsContract, - Query, - IndexPatternAttributes, -} from '../../../../../../src/plugins/data/public'; -import { getToastNotifications, getSavedObjectsClient } from './dependency_cache'; -import { IndexPatternSavedObject, SavedSearchSavedObject } from '../../../common/types/kibana'; - -let indexPatternCache: IndexPatternSavedObject[] = []; +import type { Query } from '../../../../../../src/plugins/data/public'; +import type { DataView, DataViewsContract } from '../../../../../../src/plugins/data_views/public'; +import type { SavedSearchSavedObject } from '../../../common/types/kibana'; +import { getToastNotifications, getSavedObjectsClient, getDataViews } from './dependency_cache'; + +let indexPatternCache: DataView[] = []; let savedSearchesCache: SavedSearchSavedObject[] = []; -let indexPatternsContract: IndexPatternsContract | null = null; +let indexPatternsContract: DataViewsContract | null = null; -export function loadIndexPatterns(indexPatterns: IndexPatternsContract) { +export async function loadIndexPatterns(indexPatterns: DataViewsContract) { indexPatternsContract = indexPatterns; - const savedObjectsClient = getSavedObjectsClient(); - return savedObjectsClient - .find({ - type: 'index-pattern', - fields: ['id', 'title', 'type', 'fields'], - perPage: 10000, - }) - .then((response) => { - indexPatternCache = response.savedObjects; - return indexPatternCache; - }); + const dataViewsContract = getDataViews(); + indexPatternCache = await dataViewsContract.find('*', 10000); + return indexPatternCache; } export function loadSavedSearches() { @@ -63,20 +50,15 @@ export function getIndexPatternsContract() { } export function getIndexPatternNames() { - return indexPatternCache.map((i) => i.attributes && i.attributes.title); + return indexPatternCache.map((i) => i.title); } export function getIndexPatternIdFromName(name: string) { - for (let j = 0; j < indexPatternCache.length; j++) { - if (indexPatternCache[j].get('title') === name) { - return indexPatternCache[j].id; - } - } - return null; + return indexPatternCache.find((i) => i.title === name)?.id ?? null; } export interface IndexPatternAndSavedSearch { savedSearch: SavedSearchSavedObject | null; - indexPattern: IIndexPattern | null; + indexPattern: DataView | null; } export async function getIndexPatternAndSavedSearch(savedSearchId: string) { const resp: IndexPatternAndSavedSearch = { @@ -106,7 +88,7 @@ export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject) { }; } -export function getIndexPatternById(id: string): Promise { +export function getIndexPatternById(id: string): Promise { if (indexPatternsContract !== null) { if (id) { return indexPatternsContract.get(id); @@ -127,7 +109,7 @@ export function getSavedSearchById(id: string): SavedSearchSavedObject | undefin * an optional flag will trigger the display a notification at the top of the page * warning that the index is not time based */ -export function timeBasedIndexCheck(indexPattern: IndexPattern, showNotification = false) { +export function timeBasedIndexCheck(indexPattern: DataView, showNotification = false) { if (!indexPattern.isTimeBased()) { if (showNotification) { const toastNotifications = getToastNotifications(); diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable.tsx index 65d26b844e960..47be6065aa99b 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable.tsx @@ -21,7 +21,7 @@ import { AnomalyChartsEmbeddableOutput, AnomalyChartsServices, } from '..'; -import type { IndexPattern } from '../../../../../../src/plugins/data/common'; +import type { DataView } from '../../../../../../src/plugins/data_views/common'; import { EmbeddableLoading } from '../common/components/embeddable_loading_fallback'; export const getDefaultExplorerChartsPanelTitle = (jobIds: JobId[]) => i18n.translate('xpack.ml.anomalyChartsEmbeddable.title', { @@ -66,7 +66,7 @@ export class AnomalyChartsEmbeddable extends Embeddable< const indices = new Set(jobs.map((j) => j.datafeed_config.indices).flat()); // Then find the index patterns assuming the index pattern title matches the index name - const indexPatterns: Record = {}; + const indexPatterns: Record = {}; for (const indexName of indices) { const response = await indexPatternsService.find(`"${indexName}"`); diff --git a/x-pack/plugins/ml/public/embeddables/types.ts b/x-pack/plugins/ml/public/embeddables/types.ts index 7c93fb31e9a6c..bf23f397fe08c 100644 --- a/x-pack/plugins/ml/public/embeddables/types.ts +++ b/x-pack/plugins/ml/public/embeddables/types.ts @@ -27,7 +27,7 @@ import { ANOMALY_SWIMLANE_EMBEDDABLE_TYPE, } from './constants'; import { MlResultsService } from '../application/services/results_service'; -import { IndexPattern } from '../../../../../src/plugins/data/common'; +import type { DataView } from '../../../../../src/plugins/data_views/common'; export interface AnomalySwimlaneEmbeddableCustomInput { jobIds: JobId[]; @@ -110,7 +110,7 @@ export type AnomalyChartsEmbeddableServices = [CoreStart, MlDependencies, Anomal export interface AnomalyChartsCustomOutput { entityFields?: EntityField[]; severity?: number; - indexPatterns?: IndexPattern[]; + indexPatterns?: DataView[]; } export type AnomalyChartsEmbeddableOutput = EmbeddableOutput & AnomalyChartsCustomOutput; export interface EditAnomalyChartsPanelContext { diff --git a/x-pack/plugins/ml/server/lib/data_views_utils.ts b/x-pack/plugins/ml/server/lib/data_views_utils.ts new file mode 100644 index 0000000000000..497404425eff8 --- /dev/null +++ b/x-pack/plugins/ml/server/lib/data_views_utils.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; + +import type { DataViewsService } from '../../../../../src/plugins/data_views/common'; +import type { PluginStart as DataViewsPluginStart } from '../../../../../src/plugins/data_views/server'; + +export type GetDataViewsService = () => Promise; + +export function getDataViewsServiceFactory( + getDataViews: () => DataViewsPluginStart | null, + savedObjectClient: SavedObjectsClientContract, + scopedClient: IScopedClusterClient +): GetDataViewsService { + const dataViews = getDataViews(); + if (dataViews === null) { + throw Error('data views service has not been initialized'); + } + + return () => dataViews.dataViewsServiceFactory(savedObjectClient, scopedClient.asInternalUser); +} diff --git a/x-pack/plugins/ml/server/lib/route_guard.ts b/x-pack/plugins/ml/server/lib/route_guard.ts index 1a066660d4ee0..b7b0568c10a31 100644 --- a/x-pack/plugins/ml/server/lib/route_guard.ts +++ b/x-pack/plugins/ml/server/lib/route_guard.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { +import type { KibanaRequest, KibanaResponseFactory, RequestHandlerContext, @@ -13,14 +13,17 @@ import { RequestHandler, SavedObjectsClientContract, } from 'kibana/server'; -import { SpacesPluginSetup } from '../../../spaces/server'; +import type { SpacesPluginSetup } from '../../../spaces/server'; import type { SecurityPluginSetup } from '../../../security/server'; import { jobSavedObjectServiceFactory, JobSavedObjectService } from '../saved_objects'; -import { MlLicense } from '../../common/license'; +import type { MlLicense } from '../../common/license'; import { MlClient, getMlClient } from '../lib/ml_client'; import type { AlertingApiRequestHandlerContext } from '../../../alerting/server'; +import type { PluginStart as DataViewsPluginStart } from '../../../../../src/plugins/data_views/server'; +import type { DataViewsService } from '../../../../../src/plugins/data_views/common'; +import { getDataViewsServiceFactory } from './data_views_utils'; type MLRequestHandlerContext = RequestHandlerContext & { alerting?: AlertingApiRequestHandlerContext; @@ -33,10 +36,12 @@ type Handler

= (handlerParams: { context: MLRequestHandlerContext; jobSavedObjectService: JobSavedObjectService; mlClient: MlClient; + getDataViewsService(): Promise; }) => ReturnType>; type GetMlSavedObjectClient = (request: KibanaRequest) => SavedObjectsClientContract | null; type GetInternalSavedObjectClient = () => SavedObjectsClientContract | null; +type GetDataViews = () => DataViewsPluginStart | null; export class RouteGuard { private _mlLicense: MlLicense; @@ -45,6 +50,7 @@ export class RouteGuard { private _spacesPlugin: SpacesPluginSetup | undefined; private _authorization: SecurityPluginSetup['authz'] | undefined; private _isMlReady: () => Promise; + private _getDataViews: GetDataViews; constructor( mlLicense: MlLicense, @@ -52,7 +58,8 @@ export class RouteGuard { getInternalSavedObject: GetInternalSavedObjectClient, spacesPlugin: SpacesPluginSetup | undefined, authorization: SecurityPluginSetup['authz'] | undefined, - isMlReady: () => Promise + isMlReady: () => Promise, + getDataViews: GetDataViews ) { this._mlLicense = mlLicense; this._getMlSavedObjectClient = getSavedObject; @@ -60,6 +67,7 @@ export class RouteGuard { this._spacesPlugin = spacesPlugin; this._authorization = authorization; this._isMlReady = isMlReady; + this._getDataViews = getDataViews; } public fullLicenseAPIGuard(handler: Handler) { @@ -79,6 +87,7 @@ export class RouteGuard { return response.forbidden(); } + const client = context.core.elasticsearch.client; const mlSavedObjectClient = this._getMlSavedObjectClient(request); const internalSavedObjectsClient = this._getInternalSavedObjectClient(); if (mlSavedObjectClient === null || internalSavedObjectsClient === null) { @@ -94,7 +103,6 @@ export class RouteGuard { this._authorization, this._isMlReady ); - const client = context.core.elasticsearch.client; return handler({ client, @@ -103,6 +111,11 @@ export class RouteGuard { context, jobSavedObjectService, mlClient: getMlClient(client, jobSavedObjectService), + getDataViewsService: getDataViewsServiceFactory( + this._getDataViews, + context.core.savedObjects.client, + client + ), }); }; } diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/index_patterns.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/index_patterns.ts index 15b4cfa5be8b1..568be4197baf8 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/index_patterns.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/index_patterns.ts @@ -5,29 +5,19 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; -import { IndexPatternAttributes } from 'src/plugins/data/server'; +import { DataViewsService } from '../../../../../../src/plugins/data_views/common'; export class IndexPatternHandler { - constructor(private savedObjectsClient: SavedObjectsClientContract) {} + constructor(private dataViewService: DataViewsService) {} // returns a id based on an index pattern name async getIndexPatternId(indexName: string) { - const response = await this.savedObjectsClient.find({ - type: 'index-pattern', - perPage: 10, - search: `"${indexName}"`, - searchFields: ['title'], - fields: ['title'], - }); - - const ip = response.saved_objects.find( - (obj) => obj.attributes.title.toLowerCase() === indexName.toLowerCase() + const dv = (await this.dataViewService.find(indexName)).find( + ({ title }) => title === indexName ); - - return ip?.id; + return dv?.id; } async deleteIndexPatternById(indexId: string) { - return await this.savedObjectsClient.delete('index-pattern', indexId); + return await this.dataViewService.delete(indexId); } } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.test.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.test.ts index 8ddb805af2033..e853d5de5899d 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.test.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.test.ts @@ -5,11 +5,16 @@ * 2.0. */ -import { SavedObjectsClientContract, KibanaRequest, IScopedClusterClient } from 'kibana/server'; -import { Module } from '../../../common/types/modules'; +import type { + SavedObjectsClientContract, + KibanaRequest, + IScopedClusterClient, +} from 'kibana/server'; +import type { DataViewsService } from '../../../../../../src/plugins/data_views/common'; +import type { Module } from '../../../common/types/modules'; import { DataRecognizer } from '../data_recognizer'; import type { MlClient } from '../../lib/ml_client'; -import { JobSavedObjectService } from '../../saved_objects'; +import type { JobSavedObjectService } from '../../saved_objects'; const callAs = () => Promise.resolve({ body: {} }); @@ -28,6 +33,7 @@ describe('ML - data recognizer', () => { find: jest.fn(), bulkCreate: jest.fn(), } as unknown as SavedObjectsClientContract, + { find: jest.fn() } as unknown as DataViewsService, {} as JobSavedObjectService, { headers: { authorization: '' } } as KibanaRequest ); diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index f9c609803217e..711ec0d458f27 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -8,17 +8,21 @@ import fs from 'fs'; import Boom from '@hapi/boom'; import numeral from '@elastic/numeral'; -import { KibanaRequest, IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; +import type { + KibanaRequest, + IScopedClusterClient, + SavedObjectsClientContract, +} from 'kibana/server'; import moment from 'moment'; -import { IndexPatternAttributes } from 'src/plugins/data/server'; import { merge } from 'lodash'; -import { AnalysisLimits } from '../../../common/types/anomaly_detection_jobs'; +import type { DataViewsService } from '../../../../../../src/plugins/data_views/common'; +import type { AnalysisLimits } from '../../../common/types/anomaly_detection_jobs'; import { getAuthorizationHeader } from '../../lib/request_authorization'; -import { MlInfoResponse } from '../../../common/types/ml_server_info'; +import type { MlInfoResponse } from '../../../common/types/ml_server_info'; import type { MlClient } from '../../lib/ml_client'; import { ML_MODULE_SAVED_OBJECT_TYPE } from '../../../common/types/saved_objects'; -import { +import type { KibanaObjects, KibanaObjectConfig, ModuleDatafeed, @@ -35,8 +39,8 @@ import { DataRecognizerConfigResponse, GeneralDatafeedsOverride, JobSpecificOverride, - isGeneralJobOverride, } from '../../../common/types/modules'; +import { isGeneralJobOverride } from '../../../common/types/modules'; import { getLatestDataOrBucketTimestamp, prefixDatafeedId, @@ -47,10 +51,10 @@ import { calculateModelMemoryLimitProvider } from '../calculate_model_memory_lim import { fieldsServiceProvider } from '../fields_service'; import { jobServiceProvider } from '../job_service'; import { resultsServiceProvider } from '../results_service'; -import { JobExistResult, JobStat } from '../../../common/types/data_recognizer'; -import { MlJobsStatsResponse } from '../../../common/types/job_service'; -import { Datafeed } from '../../../common/types/anomaly_detection_jobs'; -import { JobSavedObjectService } from '../../saved_objects'; +import type { JobExistResult, JobStat } from '../../../common/types/data_recognizer'; +import type { MlJobsStatsResponse } from '../../../common/types/job_service'; +import type { Datafeed } from '../../../common/types/anomaly_detection_jobs'; +import type { JobSavedObjectService } from '../../saved_objects'; import { isDefined } from '../../../common/types/guards'; import { isPopulatedObject } from '../../../common/util/object_utils'; @@ -110,6 +114,7 @@ export class DataRecognizer { private _mlClient: MlClient; private _savedObjectsClient: SavedObjectsClientContract; private _jobSavedObjectService: JobSavedObjectService; + private _dataViewsService: DataViewsService; private _request: KibanaRequest; private _authorizationHeader: object; @@ -130,12 +135,14 @@ export class DataRecognizer { mlClusterClient: IScopedClusterClient, mlClient: MlClient, savedObjectsClient: SavedObjectsClientContract, + dataViewsService: DataViewsService, jobSavedObjectService: JobSavedObjectService, request: KibanaRequest ) { this._client = mlClusterClient; this._mlClient = mlClient; this._savedObjectsClient = savedObjectsClient; + this._dataViewsService = dataViewsService; this._jobSavedObjectService = jobSavedObjectService; this._request = request; this._authorizationHeader = getAuthorizationHeader(request); @@ -615,22 +622,11 @@ export class DataRecognizer { return results; } - private async _loadIndexPatterns() { - return await this._savedObjectsClient.find({ - type: 'index-pattern', - perPage: 1000, - }); - } - // returns a id based on an index pattern name - private async _getIndexPatternId(name: string) { + private async _getIndexPatternId(name: string): Promise { try { - const indexPatterns = await this._loadIndexPatterns(); - if (indexPatterns === undefined || indexPatterns.saved_objects === undefined) { - return; - } - const ip = indexPatterns.saved_objects.find((i) => i.attributes.title === name); - return ip !== undefined ? ip.id : undefined; + const dataViews = await this._dataViewsService.find(name); + return dataViews.find((d) => d.title === name)?.id; } catch (error) { mlLog.warn(`Error loading index patterns, ${error}`); return; @@ -1387,3 +1383,21 @@ export class DataRecognizer { } } } + +export function dataRecognizerFactory( + client: IScopedClusterClient, + mlClient: MlClient, + savedObjectsClient: SavedObjectsClientContract, + dataViewsService: DataViewsService, + jobSavedObjectService: JobSavedObjectService, + request: KibanaRequest +) { + return new DataRecognizer( + client, + mlClient, + savedObjectsClient, + dataViewsService, + jobSavedObjectService, + request + ); +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/index.ts b/x-pack/plugins/ml/server/models/data_recognizer/index.ts index 27c45726ae249..fbddf17a50ede 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/index.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export { DataRecognizer, RecognizeResult } from './data_recognizer'; +export { DataRecognizer, RecognizeResult, dataRecognizerFactory } from './data_recognizer'; diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/responses/data_view_rollup_cloudwatch.json b/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/responses/data_view_rollup_cloudwatch.json new file mode 100644 index 0000000000000..a6aeb81431532 --- /dev/null +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/responses/data_view_rollup_cloudwatch.json @@ -0,0 +1,151 @@ +[ + { + "id": "d6ac99b0-2777-11ec-8e1c-aba6d0767aaa", + "title": "cloud_roll_index", + "fieldFormatMap": {}, + "typeMeta": { + "params": { + "rollup_index": "cloud_roll_index" + }, + "aggs": { + "date_histogram": { + "@timestamp": { + "agg": "date_histogram", + "fixed_interval": "5m", + "time_zone": "UTC" + } + } + } + }, + "fields": [ + { + "count": 0, + "name": "_source", + "type": "_source", + "esTypes": [ + "_source" + ], + "scripted": false, + "searchable": false, + "aggregatable": false, + "readFromDocValues": false + }, + { + "count": 0, + "name": "_id", + "type": "string", + "esTypes": [ + "_id" + ], + "scripted": false, + "searchable": true, + "aggregatable": false, + "readFromDocValues": false + }, + { + "count": 0, + "name": "_type", + "type": "string", + "scripted": false, + "searchable": false, + "aggregatable": false, + "readFromDocValues": false + }, + { + "count": 0, + "name": "_index", + "type": "string", + "esTypes": [ + "_index" + ], + "scripted": false, + "searchable": true, + "aggregatable": true, + "readFromDocValues": false + }, + { + "count": 0, + "name": "_score", + "type": "number", + "scripted": false, + "searchable": false, + "aggregatable": false, + "readFromDocValues": false + }, + { + "count": 0, + "name": "@timestamp", + "type": "date", + "esTypes": [ + "date" + ], + "scripted": false, + "searchable": true, + "aggregatable": true, + "readFromDocValues": true + } + ], + "timeFieldName": "@timestamp", + "type": "rollup", + "metaFields": [ + "_source", + "_id", + "_type", + "_index", + "_score" + ], + "version": "WzY5NjEsNF0=", + "originalSavedObjectBody": { + "fieldAttrs": "{}", + "title": "cloud_roll_index", + "timeFieldName": "@timestamp", + "fields": "[]", + "type": "rollup", + "typeMeta": "{\"params\":{\"rollup_index\":\"cloud_roll_index\"},\"aggs\":{\"date_histogram\":{\"@timestamp\":{\"agg\":\"date_histogram\",\"fixed_interval\":\"5m\",\"time_zone\":\"UTC\"}}}}", + "runtimeFieldMap": "{}" + }, + "shortDotsEnable": false, + "fieldFormats": { + "fieldFormats": {}, + "defaultMap": { + "ip": { + "id": "ip", + "params": {} + }, + "date": { + "id": "date", + "params": {} + }, + "date_nanos": { + "id": "date_nanos", + "params": {}, + "es": true + }, + "number": { + "id": "number", + "params": {} + }, + "boolean": { + "id": "boolean", + "params": {} + }, + "histogram": { + "id": "histogram", + "params": {} + }, + "_source": { + "id": "_source", + "params": {} + }, + "_default_": { + "id": "string", + "params": {} + } + }, + "metaParamsOptions": {} + }, + "fieldAttrs": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false + } +] diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/responses/kibana_saved_objects.json b/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/responses/kibana_saved_objects.json deleted file mode 100644 index 9e2af76264231..0000000000000 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/responses/kibana_saved_objects.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "page": 1, - "per_page": 1000, - "total": 4, - "saved_objects": [ - { - "type": "index-pattern", - "id": "be0eebe0-65ac-11e9-aa86-0793be5f3670", - "attributes": { - "title": "farequote-*" - }, - "references": [], - "migrationVersion": { - "index-pattern": "6.5.0" - }, - "updated_at": "2019-04-23T09:47:02.203Z", - "version": "WzcsMV0=" - }, - { - "type": "index-pattern", - "id": "be14ceb0-66b1-11e9-91c9-ffa52374d341", - "attributes": { - "typeMeta": "{\"params\":{\"rollup_index\":\"cloud_roll_index\"},\"aggs\":{\"histogram\":{\"NetworkOut\":{\"agg\":\"histogram\",\"interval\":5},\"CPUUtilization\":{\"agg\":\"histogram\",\"interval\":5},\"NetworkIn\":{\"agg\":\"histogram\",\"interval\":5}},\"avg\":{\"NetworkOut\":{\"agg\":\"avg\"},\"CPUUtilization\":{\"agg\":\"avg\"},\"NetworkIn\":{\"agg\":\"avg\"},\"DiskReadBytes\":{\"agg\":\"avg\"}},\"min\":{\"NetworkOut\":{\"agg\":\"min\"},\"NetworkIn\":{\"agg\":\"min\"}},\"value_count\":{\"NetworkOut\":{\"agg\":\"value_count\"},\"DiskReadBytes\":{\"agg\":\"value_count\"},\"CPUUtilization\":{\"agg\":\"value_count\"},\"NetworkIn\":{\"agg\":\"value_count\"}},\"max\":{\"CPUUtilization\":{\"agg\":\"max\"},\"DiskReadBytes\":{\"agg\":\"max\"}},\"date_histogram\":{\"@timestamp\":{\"agg\":\"date_histogram\",\"delay\":\"1d\",\"fixed_interval\":\"5m\",\"time_zone\":\"UTC\"}},\"terms\":{\"instance\":{\"agg\":\"terms\"},\"sourcetype.keyword\":{\"agg\":\"terms\"},\"region\":{\"agg\":\"terms\"}},\"sum\":{\"DiskReadBytes\":{\"agg\":\"sum\"},\"NetworkOut\":{\"agg\":\"sum\"}}}}", - "title": "cloud_roll_index", - "type": "rollup" - }, - "references": [], - "migrationVersion": { - "index-pattern": "6.5.0" - }, - "updated_at": "2019-04-24T16:55:20.550Z", - "version": "Wzc0LDJd" - } - ] -} diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts b/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts index b838165826da1..a25b3183362b3 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts @@ -5,12 +5,12 @@ * 2.0. */ -import { estypes } from '@elastic/elasticsearch'; -import { IScopedClusterClient } from 'kibana/server'; import { cloneDeep } from 'lodash'; -import { SavedObjectsClientContract } from 'kibana/server'; -import { Field, FieldId, NewJobCaps, RollupFields } from '../../../../common/types/fields'; +import { estypes } from '@elastic/elasticsearch'; +import type { IScopedClusterClient } from 'kibana/server'; +import type { Field, FieldId, NewJobCaps, RollupFields } from '../../../../common/types/fields'; import { ES_FIELD_TYPES } from '../../../../../../../src/plugins/data/common'; +import type { DataViewsService } from '../../../../../../../src/plugins/data_views/common'; import { combineFieldsAndAggs } from '../../../../common/util/fields_utils'; import { rollupServiceProvider } from './rollup'; import { aggregations, mlOnlyAggregations } from '../../../../common/constants/aggregation_types'; @@ -38,27 +38,27 @@ export function fieldServiceProvider( indexPattern: string, isRollup: boolean, client: IScopedClusterClient, - savedObjectsClient: SavedObjectsClientContract + dataViewsService: DataViewsService ) { - return new FieldsService(indexPattern, isRollup, client, savedObjectsClient); + return new FieldsService(indexPattern, isRollup, client, dataViewsService); } class FieldsService { private _indexPattern: string; private _isRollup: boolean; private _mlClusterClient: IScopedClusterClient; - private _savedObjectsClient: SavedObjectsClientContract; + private _dataViewsService: DataViewsService; constructor( indexPattern: string, isRollup: boolean, client: IScopedClusterClient, - savedObjectsClient: SavedObjectsClientContract + dataViewsService: DataViewsService ) { this._indexPattern = indexPattern; this._isRollup = isRollup; this._mlClusterClient = client; - this._savedObjectsClient = savedObjectsClient; + this._dataViewsService = dataViewsService; } private async loadFieldCaps(): Promise { @@ -111,7 +111,7 @@ class FieldsService { const rollupService = await rollupServiceProvider( this._indexPattern, this._mlClusterClient, - this._savedObjectsClient + this._dataViewsService ); const rollupConfigs: estypes.RollupGetRollupCapabilitiesRollupCapabilitySummary[] | null = await rollupService.getRollupJobs(); diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.test.ts b/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.test.ts index 3eb2ba5bbaced..c0f270f1df96c 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.test.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.test.ts @@ -10,7 +10,7 @@ import { newJobCapsProvider } from './index'; import farequoteFieldCaps from './__mocks__/responses/farequote_field_caps.json'; import cloudwatchFieldCaps from './__mocks__/responses/cloudwatch_field_caps.json'; import rollupCaps from './__mocks__/responses/rollup_caps.json'; -import kibanaSavedObjects from './__mocks__/responses/kibana_saved_objects.json'; +import dataView from './__mocks__/responses/data_view_rollup_cloudwatch.json'; import farequoteJobCaps from './__mocks__/results/farequote_job_caps.json'; import farequoteJobCapsEmpty from './__mocks__/results/farequote_job_caps_empty.json'; @@ -19,7 +19,7 @@ import cloudwatchJobCaps from './__mocks__/results/cloudwatch_rollup_job_caps.js describe('job_service - job_caps', () => { let mlClusterClientNonRollupMock: any; let mlClusterClientRollupMock: any; - let savedObjectsClientMock: any; + let dataViews: any; beforeEach(() => { const asNonRollupMock = { @@ -41,9 +41,9 @@ describe('job_service - job_caps', () => { asInternalUser: callAsRollupMock, }; - savedObjectsClientMock = { + dataViews = { async find() { - return Promise.resolve(kibanaSavedObjects); + return Promise.resolve(dataView); }, }; }); @@ -53,7 +53,7 @@ describe('job_service - job_caps', () => { const indexPattern = 'farequote-*'; const isRollup = false; const { newJobCaps } = newJobCapsProvider(mlClusterClientNonRollupMock); - const response = await newJobCaps(indexPattern, isRollup, savedObjectsClientMock); + const response = await newJobCaps(indexPattern, isRollup, dataViews); expect(response).toEqual(farequoteJobCaps); }); @@ -61,7 +61,7 @@ describe('job_service - job_caps', () => { const indexPattern = 'farequote-*'; const isRollup = true; const { newJobCaps } = newJobCapsProvider(mlClusterClientNonRollupMock); - const response = await newJobCaps(indexPattern, isRollup, savedObjectsClientMock); + const response = await newJobCaps(indexPattern, isRollup, dataViews); expect(response).toEqual(farequoteJobCapsEmpty); }); }); @@ -71,7 +71,7 @@ describe('job_service - job_caps', () => { const indexPattern = 'cloud_roll_index'; const isRollup = true; const { newJobCaps } = newJobCapsProvider(mlClusterClientRollupMock); - const response = await newJobCaps(indexPattern, isRollup, savedObjectsClientMock); + const response = await newJobCaps(indexPattern, isRollup, dataViews); expect(response).toEqual(cloudwatchJobCaps); }); @@ -79,7 +79,7 @@ describe('job_service - job_caps', () => { const indexPattern = 'cloud_roll_index'; const isRollup = false; const { newJobCaps } = newJobCapsProvider(mlClusterClientRollupMock); - const response = await newJobCaps(indexPattern, isRollup, savedObjectsClientMock); + const response = await newJobCaps(indexPattern, isRollup, dataViews); expect(response).not.toEqual(cloudwatchJobCaps); }); }); diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts b/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts index 6444f9ae3f61a..bab4fb31aa1a9 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts @@ -5,18 +5,19 @@ * 2.0. */ -import { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; +import type { IScopedClusterClient } from 'kibana/server'; +import type { DataViewsService } from '../../../../../../../src/plugins/data_views/common'; +import type { Aggregation, Field, NewJobCapsResponse } from '../../../../common/types/fields'; import { _DOC_COUNT } from '../../../../common/constants/field_types'; -import { Aggregation, Field, NewJobCapsResponse } from '../../../../common/types/fields'; import { fieldServiceProvider } from './field_service'; export function newJobCapsProvider(client: IScopedClusterClient) { async function newJobCaps( indexPattern: string, isRollup: boolean = false, - savedObjectsClient: SavedObjectsClientContract + dataViewsService: DataViewsService ): Promise { - const fieldService = fieldServiceProvider(indexPattern, isRollup, client, savedObjectsClient); + const fieldService = fieldServiceProvider(indexPattern, isRollup, client, dataViewsService); const { aggs, fields } = await fieldService.getData(); convertForStringify(aggs, fields); diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/rollup.ts b/x-pack/plugins/ml/server/models/job_service/new_job_caps/rollup.ts index 72408b7f9c534..87504a1bc0e10 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/rollup.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/rollup.ts @@ -6,11 +6,12 @@ */ import { estypes } from '@elastic/elasticsearch'; -import { IScopedClusterClient } from 'kibana/server'; -import { SavedObject } from 'kibana/server'; -import { IndexPatternAttributes } from 'src/plugins/data/server'; -import { SavedObjectsClientContract } from 'kibana/server'; -import { RollupFields } from '../../../../common/types/fields'; +import type { IScopedClusterClient } from 'kibana/server'; +import type { + DataViewsService, + DataView, +} from '../../../../../../../src/plugins/data_views/common'; +import type { RollupFields } from '../../../../common/types/fields'; export interface RollupJob { job_id: string; @@ -22,17 +23,19 @@ export interface RollupJob { export async function rollupServiceProvider( indexPattern: string, { asCurrentUser }: IScopedClusterClient, - savedObjectsClient: SavedObjectsClientContract + dataViewsService: DataViewsService ) { - const rollupIndexPatternObject = await loadRollupIndexPattern(indexPattern, savedObjectsClient); + const rollupIndexPatternObject = await loadRollupIndexPattern(indexPattern, dataViewsService); let jobIndexPatterns: string[] = [indexPattern]; async function getRollupJobs(): Promise< estypes.RollupGetRollupCapabilitiesRollupCapabilitySummary[] | null > { - if (rollupIndexPatternObject !== null) { - const parsedTypeMetaData = JSON.parse(rollupIndexPatternObject.attributes.typeMeta!); - const rollUpIndex: string = parsedTypeMetaData.params.rollup_index; + if ( + rollupIndexPatternObject !== null && + rollupIndexPatternObject.typeMeta?.params !== undefined + ) { + const rollUpIndex: string = rollupIndexPatternObject.typeMeta.params.rollup_index; const { body: rollupCaps } = await asCurrentUser.rollup.getRollupIndexCaps({ index: rollUpIndex, }); @@ -60,21 +63,12 @@ export async function rollupServiceProvider( async function loadRollupIndexPattern( indexPattern: string, - savedObjectsClient: SavedObjectsClientContract -): Promise | null> { - const resp = await savedObjectsClient.find({ - type: 'index-pattern', - fields: ['title', 'type', 'typeMeta'], - perPage: 1000, - }); - - const obj = resp.saved_objects.find( - (r) => - r.attributes && - r.attributes.type === 'rollup' && - r.attributes.title === indexPattern && - r.attributes.typeMeta !== undefined + dataViewsService: DataViewsService +): Promise { + const resp = await dataViewsService.find('*', 10000); + const obj = resp.find( + (dv) => dv.type === 'rollup' && dv.title === indexPattern && dv.typeMeta !== undefined ); - return obj || null; + return obj ?? null; } diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index 3876193cfbe39..efa61593655ac 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { +import type { CoreSetup, CoreStart, Plugin, @@ -21,10 +21,11 @@ import { } from 'kibana/server'; import type { SecurityPluginSetup } from '../../security/server'; import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server'; -import { PluginsSetup, PluginsStart, RouteInitialization } from './types'; -import { SpacesPluginSetup } from '../../spaces/server'; +import type { PluginStart as DataViewsPluginStart } from '../../../../src/plugins/data_views/server'; +import type { PluginsSetup, PluginsStart, RouteInitialization } from './types'; +import type { SpacesPluginSetup } from '../../spaces/server'; import { PLUGIN_ID } from '../common/constants/app'; -import { MlCapabilities } from '../common/types/capabilities'; +import type { MlCapabilities } from '../common/types/capabilities'; import { initMlServerLog } from './lib/log'; import { initSampleDataSets } from './lib/sample_data_sets'; @@ -78,6 +79,7 @@ export class MlServerPlugin private savedObjectsStart: SavedObjectsServiceStart | null = null; private spacesPlugin: SpacesPluginSetup | undefined; private security: SecurityPluginSetup | undefined; + private dataViews: DataViewsPluginStart | null = null; private isMlReady: Promise; private setMlReady: () => void = () => {}; private readonly kibanaIndexConfig: SharedGlobalConfig; @@ -156,7 +158,8 @@ export class MlServerPlugin getInternalSavedObjectsClient, plugins.spaces, plugins.security?.authz, - () => this.isMlReady + () => this.isMlReady, + () => this.dataViews ), mlLicense: this.mlLicense, }; @@ -173,6 +176,13 @@ export class MlServerPlugin ? () => coreSetup.getStartServices().then(([, { spaces }]) => spaces!) : undefined; + const getDataViews = () => { + if (this.dataViews === null) { + throw Error('Data views plugin not initialized'); + } + return this.dataViews; + }; + annotationRoutes(routeInit, plugins.security); calendars(routeInit); dataFeedRoutes(routeInit); @@ -211,6 +221,7 @@ export class MlServerPlugin () => getInternalSavedObjectsClient(), () => this.uiSettings, () => this.fieldsFormat, + getDataViews, () => this.isMlReady ); @@ -236,6 +247,7 @@ export class MlServerPlugin this.capabilities = coreStart.capabilities; this.clusterClient = coreStart.elasticsearch.client; this.savedObjectsStart = coreStart.savedObjects; + this.dataViews = plugins.dataViews; // check whether the job saved objects exist // and create them if needed. diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts index bedc70566a62f..bbfcc0fd5e500 100644 --- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts +++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts @@ -5,12 +5,12 @@ * 2.0. */ -import { RequestHandlerContext, IScopedClusterClient } from 'kibana/server'; +import type { IScopedClusterClient } from 'kibana/server'; import { wrapError } from '../client/error_wrapper'; import { analyticsAuditMessagesProvider } from '../models/data_frame_analytics/analytics_audit_messages'; -import { RouteInitialization } from '../types'; +import type { RouteInitialization } from '../types'; import { JOB_MAP_NODE_TYPES } from '../../common/constants/data_frame_analytics'; -import { Field, Aggregation } from '../../common/types/fields'; +import type { Field, Aggregation } from '../../common/types/fields'; import { dataAnalyticsJobConfigSchema, dataAnalyticsJobUpdateSchema, @@ -25,22 +25,26 @@ import { analyticsNewJobCapsParamsSchema, analyticsNewJobCapsQuerySchema, } from './schemas/data_analytics_schema'; -import { GetAnalyticsMapArgs, ExtendAnalyticsMapArgs } from '../models/data_frame_analytics/types'; +import type { + GetAnalyticsMapArgs, + ExtendAnalyticsMapArgs, +} from '../models/data_frame_analytics/types'; import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns'; import { AnalyticsManager } from '../models/data_frame_analytics/analytics_manager'; import { validateAnalyticsJob } from '../models/data_frame_analytics/validation'; import { fieldServiceProvider } from '../models/job_service/new_job_caps/field_service'; -import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics'; +import type { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics'; import { getAuthorizationHeader } from '../lib/request_authorization'; import type { MlClient } from '../lib/ml_client'; +import type { DataViewsService } from '../../../../../src/plugins/data_views/common'; -function getIndexPatternId(context: RequestHandlerContext, patternName: string) { - const iph = new IndexPatternHandler(context.core.savedObjects.client); +function getIndexPatternId(dataViewsService: DataViewsService, patternName: string) { + const iph = new IndexPatternHandler(dataViewsService); return iph.getIndexPatternId(patternName); } -function deleteDestIndexPatternById(context: RequestHandlerContext, indexPatternId: string) { - const iph = new IndexPatternHandler(context.core.savedObjects.client); +function deleteDestIndexPatternById(dataViewsService: DataViewsService, indexPatternId: string) { + const iph = new IndexPatternHandler(dataViewsService); return iph.deleteIndexPatternById(indexPatternId); } @@ -374,86 +378,89 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense, routeGuard }: Rout tags: ['access:ml:canDeleteDataFrameAnalytics'], }, }, - routeGuard.fullLicenseAPIGuard(async ({ mlClient, client, request, response, context }) => { - try { - const { analyticsId } = request.params; - const { deleteDestIndex, deleteDestIndexPattern } = request.query; - let destinationIndex: string | undefined; - const analyticsJobDeleted: DeleteDataFrameAnalyticsWithIndexStatus = { success: false }; - const destIndexDeleted: DeleteDataFrameAnalyticsWithIndexStatus = { success: false }; - const destIndexPatternDeleted: DeleteDataFrameAnalyticsWithIndexStatus = { - success: false, - }; - + routeGuard.fullLicenseAPIGuard( + async ({ mlClient, client, request, response, getDataViewsService }) => { try { - // Check if analyticsId is valid and get destination index - const { body } = await mlClient.getDataFrameAnalytics({ - id: analyticsId, - }); - if (Array.isArray(body.data_frame_analytics) && body.data_frame_analytics.length > 0) { - destinationIndex = body.data_frame_analytics[0].dest.index; + const { analyticsId } = request.params; + const { deleteDestIndex, deleteDestIndexPattern } = request.query; + let destinationIndex: string | undefined; + const analyticsJobDeleted: DeleteDataFrameAnalyticsWithIndexStatus = { success: false }; + const destIndexDeleted: DeleteDataFrameAnalyticsWithIndexStatus = { success: false }; + const destIndexPatternDeleted: DeleteDataFrameAnalyticsWithIndexStatus = { + success: false, + }; + + try { + // Check if analyticsId is valid and get destination index + const { body } = await mlClient.getDataFrameAnalytics({ + id: analyticsId, + }); + if (Array.isArray(body.data_frame_analytics) && body.data_frame_analytics.length > 0) { + destinationIndex = body.data_frame_analytics[0].dest.index; + } + } catch (e) { + // exist early if the job doesn't exist + return response.customError(wrapError(e)); } - } catch (e) { - // exist early if the job doesn't exist - return response.customError(wrapError(e)); - } - if (deleteDestIndex || deleteDestIndexPattern) { - // If user checks box to delete the destinationIndex associated with the job - if (destinationIndex && deleteDestIndex) { - // Verify if user has privilege to delete the destination index - const userCanDeleteDestIndex = await userCanDeleteIndex(client, destinationIndex); - // If user does have privilege to delete the index, then delete the index - if (userCanDeleteDestIndex) { - try { - await client.asCurrentUser.indices.delete({ - index: destinationIndex, - }); - destIndexDeleted.success = true; - } catch ({ body }) { - destIndexDeleted.error = body; + if (deleteDestIndex || deleteDestIndexPattern) { + // If user checks box to delete the destinationIndex associated with the job + if (destinationIndex && deleteDestIndex) { + // Verify if user has privilege to delete the destination index + const userCanDeleteDestIndex = await userCanDeleteIndex(client, destinationIndex); + // If user does have privilege to delete the index, then delete the index + if (userCanDeleteDestIndex) { + try { + await client.asCurrentUser.indices.delete({ + index: destinationIndex, + }); + destIndexDeleted.success = true; + } catch ({ body }) { + destIndexDeleted.error = body; + } + } else { + return response.forbidden(); } - } else { - return response.forbidden(); } - } - // Delete the index pattern if there's an index pattern that matches the name of dest index - if (destinationIndex && deleteDestIndexPattern) { - try { - const indexPatternId = await getIndexPatternId(context, destinationIndex); - if (indexPatternId) { - await deleteDestIndexPatternById(context, indexPatternId); + // Delete the index pattern if there's an index pattern that matches the name of dest index + if (destinationIndex && deleteDestIndexPattern) { + try { + const dataViewsService = await getDataViewsService(); + const indexPatternId = await getIndexPatternId(dataViewsService, destinationIndex); + if (indexPatternId) { + await deleteDestIndexPatternById(dataViewsService, indexPatternId); + } + destIndexPatternDeleted.success = true; + } catch (deleteDestIndexPatternError) { + destIndexPatternDeleted.error = deleteDestIndexPatternError; } - destIndexPatternDeleted.success = true; - } catch (deleteDestIndexPatternError) { - destIndexPatternDeleted.error = deleteDestIndexPatternError; } } - } - // Grab the target index from the data frame analytics job id - // Delete the data frame analytics + // Grab the target index from the data frame analytics job id + // Delete the data frame analytics - try { - await mlClient.deleteDataFrameAnalytics({ - id: analyticsId, + try { + await mlClient.deleteDataFrameAnalytics({ + id: analyticsId, + }); + analyticsJobDeleted.success = true; + } catch ({ body }) { + analyticsJobDeleted.error = body; + } + const results = { + analyticsJobDeleted, + destIndexDeleted, + destIndexPatternDeleted, + }; + return response.ok({ + body: results, }); - analyticsJobDeleted.success = true; - } catch ({ body }) { - analyticsJobDeleted.error = body; + } catch (e) { + return response.customError(wrapError(e)); } - const results = { - analyticsJobDeleted, - destIndexDeleted, - destIndexPatternDeleted, - }; - return response.ok({ - body: results, - }); - } catch (e) { - return response.customError(wrapError(e)); } - }) + ) ); /** @@ -716,17 +723,12 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense, routeGuard }: Rout tags: ['access:ml:canGetJobs'], }, }, - routeGuard.fullLicenseAPIGuard(async ({ client, request, response, context }) => { + routeGuard.fullLicenseAPIGuard(async ({ client, request, response, getDataViewsService }) => { try { const { indexPattern } = request.params; const isRollup = request.query?.rollup === 'true'; - const savedObjectsClient = context.core.savedObjects.client; - const fieldService = fieldServiceProvider( - indexPattern, - isRollup, - client, - savedObjectsClient - ); + const dataViewsService = await getDataViewsService(); + const fieldService = fieldServiceProvider(indexPattern, isRollup, client, dataViewsService); const { fields, aggs } = await fieldService.getData(true); convertForStringify(aggs, fields); diff --git a/x-pack/plugins/ml/server/routes/job_service.ts b/x-pack/plugins/ml/server/routes/job_service.ts index da115a224d19e..15b0b4449590c 100644 --- a/x-pack/plugins/ml/server/routes/job_service.ts +++ b/x-pack/plugins/ml/server/routes/job_service.ts @@ -8,7 +8,7 @@ import { estypes } from '@elastic/elasticsearch'; import { schema } from '@kbn/config-schema'; import { wrapError } from '../client/error_wrapper'; -import { RouteInitialization } from '../types'; +import type { RouteInitialization } from '../types'; import { categorizationFieldExamplesSchema, basicChartSchema, @@ -32,7 +32,7 @@ import { jobIdSchema } from './schemas/anomaly_detectors_schema'; import { jobServiceProvider } from '../models/job_service'; import { categorizationExamplesProvider } from '../models/job_service/new_job'; import { getAuthorizationHeader } from '../lib/request_authorization'; -import { Datafeed, Job } from '../../common/types/anomaly_detection_jobs'; +import type { Datafeed, Job } from '../../common/types/anomaly_detection_jobs'; /** * Routes for job service @@ -535,21 +535,24 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { tags: ['access:ml:canGetJobs'], }, }, - routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response, context }) => { - try { - const { indexPattern } = request.params; - const isRollup = request.query?.rollup === 'true'; - const savedObjectsClient = context.core.savedObjects.client; - const { newJobCaps } = jobServiceProvider(client, mlClient); - const resp = await newJobCaps(indexPattern, isRollup, savedObjectsClient); - - return response.ok({ - body: resp, - }); - } catch (e) { - return response.customError(wrapError(e)); + routeGuard.fullLicenseAPIGuard( + async ({ client, mlClient, request, response, getDataViewsService }) => { + try { + const { indexPattern } = request.params; + const isRollup = request.query?.rollup === 'true'; + const { newJobCaps } = jobServiceProvider(client, mlClient); + + const dataViewsService = await getDataViewsService(); + const resp = await newJobCaps(indexPattern, isRollup, dataViewsService); + + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } } - }) + ) ); /** diff --git a/x-pack/plugins/ml/server/routes/modules.ts b/x-pack/plugins/ml/server/routes/modules.ts index 097f3f8d67652..d814e91f70ca0 100644 --- a/x-pack/plugins/ml/server/routes/modules.ts +++ b/x-pack/plugins/ml/server/routes/modules.ts @@ -5,19 +5,24 @@ * 2.0. */ -import { TypeOf } from '@kbn/config-schema'; +import type { TypeOf } from '@kbn/config-schema'; -import { IScopedClusterClient, KibanaRequest, SavedObjectsClientContract } from 'kibana/server'; -import { DatafeedOverride, JobOverride } from '../../common/types/modules'; +import type { + IScopedClusterClient, + KibanaRequest, + SavedObjectsClientContract, +} from 'kibana/server'; +import type { DataViewsService } from '../../../../../src/plugins/data_views/common'; +import type { DatafeedOverride, JobOverride } from '../../common/types/modules'; import { wrapError } from '../client/error_wrapper'; -import { DataRecognizer } from '../models/data_recognizer'; +import { dataRecognizerFactory } from '../models/data_recognizer'; import { moduleIdParamSchema, optionalModuleIdParamSchema, modulesIndexPatternTitleSchema, setupModuleBodySchema, } from './schemas/modules'; -import { RouteInitialization } from '../types'; +import type { RouteInitialization } from '../types'; import type { MlClient } from '../lib/ml_client'; import type { JobSavedObjectService } from '../saved_objects'; @@ -25,14 +30,16 @@ function recognize( client: IScopedClusterClient, mlClient: MlClient, savedObjectsClient: SavedObjectsClientContract, + dataViewsService: DataViewsService, jobSavedObjectService: JobSavedObjectService, request: KibanaRequest, indexPatternTitle: string ) { - const dr = new DataRecognizer( + const dr = dataRecognizerFactory( client, mlClient, savedObjectsClient, + dataViewsService, jobSavedObjectService, request ); @@ -43,14 +50,16 @@ function getModule( client: IScopedClusterClient, mlClient: MlClient, savedObjectsClient: SavedObjectsClientContract, + dataViewsService: DataViewsService, jobSavedObjectService: JobSavedObjectService, request: KibanaRequest, moduleId?: string ) { - const dr = new DataRecognizer( + const dr = dataRecognizerFactory( client, mlClient, savedObjectsClient, + dataViewsService, jobSavedObjectService, request ); @@ -65,6 +74,7 @@ function setup( client: IScopedClusterClient, mlClient: MlClient, savedObjectsClient: SavedObjectsClientContract, + dataViewsService: DataViewsService, jobSavedObjectService: JobSavedObjectService, request: KibanaRequest, moduleId: string, @@ -81,10 +91,11 @@ function setup( estimateModelMemory?: boolean, applyToAllSpaces?: boolean ) { - const dr = new DataRecognizer( + const dr = dataRecognizerFactory( client, mlClient, savedObjectsClient, + dataViewsService, jobSavedObjectService, request ); @@ -109,14 +120,16 @@ function dataRecognizerJobsExist( client: IScopedClusterClient, mlClient: MlClient, savedObjectsClient: SavedObjectsClientContract, + dataViewsService: DataViewsService, jobSavedObjectService: JobSavedObjectService, request: KibanaRequest, moduleId: string ) { - const dr = new DataRecognizer( + const dr = dataRecognizerFactory( client, mlClient, savedObjectsClient, + dataViewsService, jobSavedObjectService, request ); @@ -166,13 +179,23 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) { }, }, routeGuard.fullLicenseAPIGuard( - async ({ client, mlClient, request, response, context, jobSavedObjectService }) => { + async ({ + client, + mlClient, + request, + response, + context, + jobSavedObjectService, + getDataViewsService, + }) => { try { const { indexPatternTitle } = request.params; + const dataViewService = await getDataViewsService(); const results = await recognize( client, mlClient, context.core.savedObjects.client, + dataViewService, jobSavedObjectService, request, indexPatternTitle @@ -305,7 +328,15 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) { }, }, routeGuard.fullLicenseAPIGuard( - async ({ client, mlClient, request, response, context, jobSavedObjectService }) => { + async ({ + client, + mlClient, + request, + response, + context, + jobSavedObjectService, + getDataViewsService, + }) => { try { let { moduleId } = request.params; if (moduleId === '') { @@ -313,10 +344,12 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) { // the moduleId will be an empty string. moduleId = undefined; } + const dataViewService = await getDataViewsService(); const results = await getModule( client, mlClient, context.core.savedObjects.client, + dataViewService, jobSavedObjectService, request, moduleId @@ -482,7 +515,15 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) { }, }, routeGuard.fullLicenseAPIGuard( - async ({ client, mlClient, request, response, context, jobSavedObjectService }) => { + async ({ + client, + mlClient, + request, + response, + context, + jobSavedObjectService, + getDataViewsService, + }) => { try { const { moduleId } = request.params; @@ -501,10 +542,13 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) { applyToAllSpaces, } = request.body as TypeOf; + const dataViewService = await getDataViewsService(); + const result = await setup( client, mlClient, context.core.savedObjects.client, + dataViewService, jobSavedObjectService, request, moduleId, @@ -593,13 +637,23 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) { }, }, routeGuard.fullLicenseAPIGuard( - async ({ client, mlClient, request, response, context, jobSavedObjectService }) => { + async ({ + client, + mlClient, + request, + response, + context, + jobSavedObjectService, + getDataViewsService, + }) => { try { const { moduleId } = request.params; + const dataViewService = await getDataViewsService(); const result = await dataRecognizerJobsExist( client, mlClient, context.core.savedObjects.client, + dataViewService, jobSavedObjectService, request, moduleId diff --git a/x-pack/plugins/ml/server/shared_services/providers/modules.ts b/x-pack/plugins/ml/server/shared_services/providers/modules.ts index c86a40e4224ce..f6a6c58fadb4e 100644 --- a/x-pack/plugins/ml/server/shared_services/providers/modules.ts +++ b/x-pack/plugins/ml/server/shared_services/providers/modules.ts @@ -5,13 +5,12 @@ * 2.0. */ -import { IScopedClusterClient, KibanaRequest, SavedObjectsClientContract } from 'kibana/server'; -import { TypeOf } from '@kbn/config-schema'; -import { DataRecognizer } from '../../models/data_recognizer'; -import { GetGuards } from '../shared_services'; +import type { KibanaRequest, SavedObjectsClientContract } from 'kibana/server'; +import type { TypeOf } from '@kbn/config-schema'; +import type { PluginStart as DataViewsPluginStart } from '../../../../../../src/plugins/data_views/server'; +import type { GetGuards } from '../shared_services'; +import { DataRecognizer, dataRecognizerFactory } from '../../models/data_recognizer'; import { moduleIdParamSchema, setupModuleBodySchema } from '../../routes/schemas/modules'; -import { MlClient } from '../../lib/ml_client'; -import { JobSavedObjectService } from '../../saved_objects'; export type ModuleSetupPayload = TypeOf & TypeOf; @@ -28,7 +27,10 @@ export interface ModulesProvider { }; } -export function getModulesProvider(getGuards: GetGuards): ModulesProvider { +export function getModulesProvider( + getGuards: GetGuards, + getDataViews: () => DataViewsPluginStart +): ModulesProvider { return { modulesProvider(request: KibanaRequest, savedObjectsClient: SavedObjectsClientContract) { return { @@ -36,11 +38,13 @@ export function getModulesProvider(getGuards: GetGuards): ModulesProvider { return await getGuards(request, savedObjectsClient) .isFullLicense() .hasMlCapabilities(['canGetJobs']) - .ok(async ({ scopedClient, mlClient, jobSavedObjectService }) => { + .ok(async ({ scopedClient, mlClient, jobSavedObjectService, getDataViewsService }) => { + const dataViewsService = await getDataViewsService(); const dr = dataRecognizerFactory( scopedClient, mlClient, savedObjectsClient, + dataViewsService, jobSavedObjectService, request ); @@ -51,11 +55,13 @@ export function getModulesProvider(getGuards: GetGuards): ModulesProvider { return await getGuards(request, savedObjectsClient) .isFullLicense() .hasMlCapabilities(['canGetJobs']) - .ok(async ({ scopedClient, mlClient, jobSavedObjectService }) => { + .ok(async ({ scopedClient, mlClient, jobSavedObjectService, getDataViewsService }) => { + const dataViewsService = await getDataViewsService(); const dr = dataRecognizerFactory( scopedClient, mlClient, savedObjectsClient, + dataViewsService, jobSavedObjectService, request ); @@ -66,11 +72,13 @@ export function getModulesProvider(getGuards: GetGuards): ModulesProvider { return await getGuards(request, savedObjectsClient) .isFullLicense() .hasMlCapabilities(['canGetJobs']) - .ok(async ({ scopedClient, mlClient, jobSavedObjectService }) => { + .ok(async ({ scopedClient, mlClient, jobSavedObjectService, getDataViewsService }) => { + const dataViewsService = await getDataViewsService(); const dr = dataRecognizerFactory( scopedClient, mlClient, savedObjectsClient, + dataViewsService, jobSavedObjectService, request ); @@ -81,11 +89,13 @@ export function getModulesProvider(getGuards: GetGuards): ModulesProvider { return await getGuards(request, savedObjectsClient) .isFullLicense() .hasMlCapabilities(['canCreateJob']) - .ok(async ({ scopedClient, mlClient, jobSavedObjectService }) => { + .ok(async ({ scopedClient, mlClient, jobSavedObjectService, getDataViewsService }) => { + const dataViewsService = await getDataViewsService(); const dr = dataRecognizerFactory( scopedClient, mlClient, savedObjectsClient, + dataViewsService, jobSavedObjectService, request ); @@ -109,13 +119,3 @@ export function getModulesProvider(getGuards: GetGuards): ModulesProvider { }, }; } - -function dataRecognizerFactory( - client: IScopedClusterClient, - mlClient: MlClient, - savedObjectsClient: SavedObjectsClientContract, - jobSavedObjectService: JobSavedObjectService, - request: KibanaRequest -) { - return new DataRecognizer(client, mlClient, savedObjectsClient, jobSavedObjectService, request); -} diff --git a/x-pack/plugins/ml/server/shared_services/shared_services.ts b/x-pack/plugins/ml/server/shared_services/shared_services.ts index 5c8bbffe10aed..9c8ab1e069258 100644 --- a/x-pack/plugins/ml/server/shared_services/shared_services.ts +++ b/x-pack/plugins/ml/server/shared_services/shared_services.ts @@ -5,17 +5,18 @@ * 2.0. */ -import { +import type { IClusterClient, IScopedClusterClient, SavedObjectsClientContract, UiSettingsServiceStart, } from 'kibana/server'; -import { SpacesPluginStart } from '../../../spaces/server'; +import type { SpacesPluginStart } from '../../../spaces/server'; import { KibanaRequest } from '../../.././../../src/core/server'; import { MlLicense } from '../../common/license'; import type { CloudSetup } from '../../../cloud/server'; +import type { PluginStart as DataViewsPluginStart } from '../../../../../src/plugins/data_views/server'; import type { SecurityPluginSetup } from '../../../security/server'; import { licenseChecks } from './license_checks'; import { MlSystemProvider, getMlSystemProvider } from './providers/system'; @@ -26,7 +27,7 @@ import { AnomalyDetectorsProvider, getAnomalyDetectorsProvider, } from './providers/anomaly_detectors'; -import { ResolveMlCapabilities, MlCapabilitiesKey } from '../../common/types/capabilities'; +import type { ResolveMlCapabilities, MlCapabilitiesKey } from '../../common/types/capabilities'; import { hasMlCapabilitiesProvider, HasMlCapabilities } from '../lib/capabilities'; import { MLClusterClientUninitialized, @@ -45,6 +46,7 @@ import { } from '../lib/alerts/jobs_health_service'; import type { FieldFormatsStart } from '../../../../../src/plugins/field_formats/server'; import type { FieldFormatsRegistryProvider } from '../../common/types/kibana'; +import { getDataViewsServiceFactory, GetDataViewsService } from '../lib/data_views_utils'; export type SharedServices = JobServiceProvider & AnomalyDetectorsProvider & @@ -76,6 +78,7 @@ interface OkParams { mlClient: MlClient; jobSavedObjectService: JobSavedObjectService; getFieldsFormatRegistry: FieldFormatsRegistryProvider; + getDataViewsService: GetDataViewsService; } type OkCallback = (okParams: OkParams) => any; @@ -90,6 +93,7 @@ export function createSharedServices( getInternalSavedObjectsClient: () => SavedObjectsClientContract | null, getUiSettings: () => UiSettingsServiceStart | null, getFieldsFormat: () => FieldFormatsStart | null, + getDataViews: () => DataViewsPluginStart, isMlReady: () => Promise ): { sharedServicesProviders: SharedServices; @@ -101,6 +105,7 @@ export function createSharedServices( savedObjectsClient: SavedObjectsClientContract ): Guards { const internalSavedObjectsClient = getInternalSavedObjectsClient(); + if (internalSavedObjectsClient === null) { throw new Error('Internal saved object client not initialized'); } @@ -113,7 +118,8 @@ export function createSharedServices( getSpaces !== undefined, isMlReady, getUiSettings, - getFieldsFormat + getFieldsFormat, + getDataViews ); const { @@ -122,6 +128,7 @@ export function createSharedServices( mlClient, jobSavedObjectService, getFieldsFormatRegistry, + getDataViewsService, } = getRequestItems(request); const asyncGuards: Array> = []; @@ -140,7 +147,13 @@ export function createSharedServices( }, async ok(callback: OkCallback) { await Promise.all(asyncGuards); - return callback({ scopedClient, mlClient, jobSavedObjectService, getFieldsFormatRegistry }); + return callback({ + scopedClient, + mlClient, + jobSavedObjectService, + getFieldsFormatRegistry, + getDataViewsService, + }); }, }; return guards; @@ -153,7 +166,7 @@ export function createSharedServices( sharedServicesProviders: { ...getJobServiceProvider(getGuards), ...getAnomalyDetectorsProvider(getGuards), - ...getModulesProvider(getGuards), + ...getModulesProvider(getGuards, getDataViews), ...getResultsServiceProvider(getGuards), ...getMlSystemProvider(getGuards, mlLicense, getSpaces, cloud, resolveMlCapabilities), ...getAlertingServiceProvider(getGuards), @@ -176,7 +189,8 @@ function getRequestItemsProvider( spaceEnabled: boolean, isMlReady: () => Promise, getUiSettings: () => UiSettingsServiceStart | null, - getFieldsFormat: () => FieldFormatsStart | null + getFieldsFormat: () => FieldFormatsStart | null, + getDataViews: () => DataViewsPluginStart ) { return (request: KibanaRequest) => { const getHasMlCapabilities = hasMlCapabilitiesProvider(resolveMlCapabilities); @@ -234,12 +248,20 @@ function getRequestItemsProvider( }; mlClient = getMlClient(scopedClient, jobSavedObjectService); } + + const getDataViewsService = getDataViewsServiceFactory( + getDataViews, + savedObjectsClient, + scopedClient + ); + return { hasMlCapabilities, scopedClient, mlClient, jobSavedObjectService, getFieldsFormatRegistry, + getDataViewsService, }; }; } diff --git a/x-pack/plugins/ml/server/types.ts b/x-pack/plugins/ml/server/types.ts index da83b03766af4..d5c67bf99a7a0 100644 --- a/x-pack/plugins/ml/server/types.ts +++ b/x-pack/plugins/ml/server/types.ts @@ -23,6 +23,7 @@ import type { PluginSetup as DataPluginSetup, PluginStart as DataPluginStart, } from '../../../../src/plugins/data/server'; +import type { PluginStart as DataViewsPluginStart } from '../../../../src/plugins/data_views/server'; import type { FieldFormatsSetup, FieldFormatsStart, @@ -64,6 +65,7 @@ export interface PluginsSetup { export interface PluginsStart { data: DataPluginStart; + dataViews: DataViewsPluginStart; fieldFormats: FieldFormatsStart; spaces?: SpacesPluginStart; } diff --git a/x-pack/plugins/ml/tsconfig.json b/x-pack/plugins/ml/tsconfig.json index db8fc463b0550..0c108f8b3b8a5 100644 --- a/x-pack/plugins/ml/tsconfig.json +++ b/x-pack/plugins/ml/tsconfig.json @@ -21,6 +21,7 @@ { "path": "../../../src/core/tsconfig.json" }, { "path": "../../../src/plugins/embeddable/tsconfig.json" }, { "path": "../../../src/plugins/index_pattern_management/tsconfig.json" }, + { "path": "../../../src/plugins/data_views/tsconfig.json" }, { "path": "../cloud/tsconfig.json" }, { "path": "../features/tsconfig.json" }, { "path": "../data_visualizer/tsconfig.json"}, From 877e00786d5458db746371db7442aec0bd5617cd Mon Sep 17 00:00:00 2001 From: Kaarina Tungseth Date: Wed, 13 Oct 2021 13:37:21 -0500 Subject: [PATCH 16/35] [DOCS] Removes capitalized attributes (#114849) --- docs/index.asciidoc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index e286e42f2c421..f9ed2abc4b8cf 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -13,12 +13,8 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] :es-docker-image: {es-docker-repo}:{version} :blob: {kib-repo}blob/{branch}/ :security-ref: https://www.elastic.co/community/security/ -:Data-Sources: Data Views -:Data-source: Data view :data-source: data view -:Data-sources: Data views :data-sources: data views -:A-data-source: A data view :a-data-source: a data view include::{docs-root}/shared/attributes.asciidoc[] From b96f5443d60fe802b0a0f0e6cc680a9288724ed8 Mon Sep 17 00:00:00 2001 From: Kerry Gallagher <471693+Kerry350@users.noreply.github.com> Date: Wed, 13 Oct 2021 19:39:52 +0100 Subject: [PATCH 17/35] [RAC] Change index bootstrapping strategy (#113389) * Change index bootstrapping to cater for non-additive changes only --- x-pack/plugins/rule_registry/server/config.ts | 5 +- x-pack/plugins/rule_registry/server/plugin.ts | 1 - .../rule_data_client/rule_data_client.ts | 147 ++++++++++++------ .../server/rule_data_client/types.ts | 2 +- .../server/rule_data_plugin_service/errors.ts | 14 ++ .../resource_installer.ts | 71 +++------ .../rule_data_plugin_service.ts | 3 +- 7 files changed, 138 insertions(+), 105 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/config.ts b/x-pack/plugins/rule_registry/server/config.ts index 62f29a9e06294..f112a99e59eaa 100644 --- a/x-pack/plugins/rule_registry/server/config.ts +++ b/x-pack/plugins/rule_registry/server/config.ts @@ -9,7 +9,10 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { PluginConfigDescriptor } from 'src/core/server'; export const config: PluginConfigDescriptor = { - deprecations: ({ deprecate }) => [deprecate('enabled', '8.0.0')], + deprecations: ({ deprecate, unused }) => [ + deprecate('enabled', '8.0.0'), + unused('unsafe.indexUpgrade.enabled'), + ], schema: schema.object({ enabled: schema.boolean({ defaultValue: true }), write: schema.object({ diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts index 5d1994cfd3e6d..b68f3eeb10669 100644 --- a/x-pack/plugins/rule_registry/server/plugin.ts +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -104,7 +104,6 @@ export class RuleRegistryPlugin logger, kibanaVersion, isWriteEnabled: isWriteEnabled(this.config, this.legacyConfig), - isIndexUpgradeEnabled: this.config.unsafe.indexUpgrade.enabled, getClusterClient: async () => { const deps = await startDependencies; return deps.core.elasticsearch.client.asInternalUser; diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts index 89ae479132de5..2755021e235a8 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts @@ -5,13 +5,18 @@ * 2.0. */ +import { BulkRequest } from '@elastic/elasticsearch/api/types'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { Either, isLeft } from 'fp-ts/lib/Either'; import { ElasticsearchClient } from 'kibana/server'; +import { Logger } from 'kibana/server'; import { IndexPatternsFetcher } from '../../../../../src/plugins/data/server'; -import { RuleDataWriteDisabledError } from '../rule_data_plugin_service/errors'; +import { + RuleDataWriteDisabledError, + RuleDataWriterInitializationError, +} from '../rule_data_plugin_service/errors'; import { IndexInfo } from '../rule_data_plugin_service/index_info'; import { ResourceInstaller } from '../rule_data_plugin_service/resource_installer'; import { IRuleDataClient, IRuleDataReader, IRuleDataWriter } from './types'; @@ -22,12 +27,21 @@ interface ConstructorOptions { isWriteEnabled: boolean; waitUntilReadyForReading: Promise; waitUntilReadyForWriting: Promise; + logger: Logger; } export type WaitResult = Either; export class RuleDataClient implements IRuleDataClient { - constructor(private readonly options: ConstructorOptions) {} + private _isWriteEnabled: boolean = false; + + // Writers cached by namespace + private writerCache: Map; + + constructor(private readonly options: ConstructorOptions) { + this.writeEnabled = this.options.isWriteEnabled; + this.writerCache = new Map(); + } public get indexName(): string { return this.options.indexInfo.baseName; @@ -37,8 +51,16 @@ export class RuleDataClient implements IRuleDataClient { return this.options.indexInfo.kibanaVersion; } + private get writeEnabled(): boolean { + return this._isWriteEnabled; + } + + private set writeEnabled(isEnabled: boolean) { + this._isWriteEnabled = isEnabled; + } + public isWriteEnabled(): boolean { - return this.options.isWriteEnabled; + return this.writeEnabled; } public getReader(options: { namespace?: string } = {}): IRuleDataReader { @@ -95,62 +117,89 @@ export class RuleDataClient implements IRuleDataClient { } public getWriter(options: { namespace?: string } = {}): IRuleDataWriter { - const { indexInfo, resourceInstaller } = this.options; - const namespace = options.namespace || 'default'; + const cachedWriter = this.writerCache.get(namespace); + + // There is no cached writer, so we'll install / update the namespace specific resources now. + if (!cachedWriter) { + const writerForNamespace = this.initializeWriter(namespace); + this.writerCache.set(namespace, writerForNamespace); + return writerForNamespace; + } else { + return cachedWriter; + } + } + + private initializeWriter(namespace: string): IRuleDataWriter { + const isWriteEnabled = () => this.writeEnabled; + const turnOffWrite = () => (this.writeEnabled = false); + + const { indexInfo, resourceInstaller } = this.options; const alias = indexInfo.getPrimaryAlias(namespace); - const isWriteEnabled = this.isWriteEnabled(); - const waitUntilReady = async () => { - const result = await this.options.waitUntilReadyForWriting; - if (isLeft(result)) { - throw result.left; + // Wait until both index and namespace level resources have been installed / updated. + const prepareForWriting = async () => { + if (!isWriteEnabled()) { + throw new RuleDataWriteDisabledError(); + } + + const indexLevelResourcesResult = await this.options.waitUntilReadyForWriting; + + if (isLeft(indexLevelResourcesResult)) { + throw new RuleDataWriterInitializationError( + 'index', + indexInfo.indexOptions.registrationContext, + indexLevelResourcesResult.left + ); } else { - return result.right; + try { + await resourceInstaller.installAndUpdateNamespaceLevelResources(indexInfo, namespace); + return indexLevelResourcesResult.right; + } catch (e) { + throw new RuleDataWriterInitializationError( + 'namespace', + indexInfo.indexOptions.registrationContext, + e + ); + } } }; - return { - bulk: async (request) => { - if (!isWriteEnabled) { - throw new RuleDataWriteDisabledError(); - } + const prepareForWritingResult = prepareForWriting(); - const clusterClient = await waitUntilReady(); + return { + bulk: async (request: BulkRequest) => { + return prepareForWritingResult + .then((clusterClient) => { + const requestWithDefaultParameters = { + ...request, + require_alias: true, + index: alias, + }; - const requestWithDefaultParameters = { - ...request, - require_alias: true, - index: alias, - }; - - return clusterClient.bulk(requestWithDefaultParameters).then((response) => { - if (response.body.errors) { - if ( - response.body.items.length > 0 && - (response.body.items.every( - (item) => item.index?.error?.type === 'index_not_found_exception' - ) || - response.body.items.every( - (item) => item.index?.error?.type === 'illegal_argument_exception' - )) - ) { - return resourceInstaller - .installNamespaceLevelResources(indexInfo, namespace) - .then(() => { - return clusterClient.bulk(requestWithDefaultParameters).then((retryResponse) => { - if (retryResponse.body.errors) { - throw new ResponseError(retryResponse); - } - return retryResponse; - }); - }); + return clusterClient.bulk(requestWithDefaultParameters).then((response) => { + if (response.body.errors) { + const error = new ResponseError(response); + throw error; + } + return response; + }); + }) + .catch((error) => { + if (error instanceof RuleDataWriterInitializationError) { + this.options.logger.error(error); + this.options.logger.error( + `The writer for the Rule Data Client for the ${indexInfo.indexOptions.registrationContext} registration context was not initialized properly, bulk() cannot continue, and writing will be disabled.` + ); + turnOffWrite(); + } else if (error instanceof RuleDataWriteDisabledError) { + this.options.logger.debug(`Writing is disabled, bulk() will not write any data.`); + } else { + this.options.logger.error(error); } - const error = new ResponseError(response); - throw error; - } - return response; - }); + + return undefined; + }); }, }; } diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts index 0595dbeea6dc6..7c05945a98b10 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts @@ -35,5 +35,5 @@ export interface IRuleDataReader { } export interface IRuleDataWriter { - bulk(request: BulkRequest): Promise>; + bulk(request: BulkRequest): Promise | undefined>; } diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/errors.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/errors.ts index cb5dcf8e8ae76..fe8d3b3b18d9d 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/errors.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/errors.ts @@ -5,6 +5,7 @@ * 2.0. */ +/* eslint-disable max-classes-per-file */ export class RuleDataWriteDisabledError extends Error { constructor(message?: string) { super(message); @@ -12,3 +13,16 @@ export class RuleDataWriteDisabledError extends Error { this.name = 'RuleDataWriteDisabledError'; } } + +export class RuleDataWriterInitializationError extends Error { + constructor( + resourceType: 'index' | 'namespace', + registrationContext: string, + error: string | Error + ) { + super(`There has been a catastrophic error trying to install ${resourceType} level resources for the following registration context: ${registrationContext}. + This may have been due to a non-additive change to the mappings, removal and type changes are not permitted. Full error: ${error.toString()}`); + Object.setPrototypeOf(this, new.target.prototype); + this.name = 'RuleDataWriterInitializationError'; + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts index e10bb6382ab24..160261642ff25 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts @@ -20,7 +20,6 @@ import { ecsComponentTemplate } from '../../common/assets/component_templates/ec import { defaultLifecyclePolicy } from '../../common/assets/lifecycle_policies/default_lifecycle_policy'; import { IndexInfo } from './index_info'; -import { incrementIndexName } from './utils'; const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes @@ -29,7 +28,6 @@ interface ConstructorOptions { getClusterClient: () => Promise; logger: Logger; isWriteEnabled: boolean; - isIndexUpgradeEnabled: boolean; } export class ResourceInstaller { @@ -111,12 +109,10 @@ export class ResourceInstaller { * Installs index-level resources shared between all namespaces of this index: * - custom ILM policy if it was provided * - component templates - * - attempts to update mappings of existing concrete indices */ public async installIndexLevelResources(indexInfo: IndexInfo): Promise { await this.installWithTimeout(`resources for index ${indexInfo.baseName}`, async () => { const { componentTemplates, ilmPolicy } = indexInfo.indexOptions; - const { isIndexUpgradeEnabled } = this.options; if (ilmPolicy != null) { await this.createOrUpdateLifecyclePolicy({ @@ -139,35 +135,30 @@ export class ResourceInstaller { }); }) ); - - if (isIndexUpgradeEnabled) { - // TODO: Update all existing namespaced index templates matching this index' base name - - await this.updateIndexMappings(indexInfo); - } }); } - private async updateIndexMappings(indexInfo: IndexInfo) { + private async updateIndexMappings(indexInfo: IndexInfo, namespace: string) { const { logger } = this.options; const aliases = indexInfo.basePattern; - const backingIndices = indexInfo.getPatternForBackingIndices(); + const backingIndices = indexInfo.getPatternForBackingIndices(namespace); logger.debug(`Updating mappings of existing concrete indices for ${indexInfo.baseName}`); // Find all concrete indices for all namespaces of the index. const concreteIndices = await this.fetchConcreteIndices(aliases, backingIndices); - const concreteWriteIndices = concreteIndices.filter((item) => item.isWriteIndex); - - // Update mappings of the found write indices. - await Promise.all(concreteWriteIndices.map((item) => this.updateAliasWriteIndexMapping(item))); + // Update mappings of the found indices. + await Promise.all(concreteIndices.map((item) => this.updateAliasWriteIndexMapping(item))); } + // NOTE / IMPORTANT: Please note this will update the mappings of backing indices but + // *not* the settings. This is due to the fact settings can be classed as dynamic and static, + // and static updates will fail on an index that isn't closed. New settings *will* be applied as part + // of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654 private async updateAliasWriteIndexMapping({ index, alias }: ConcreteIndexInfo) { const { logger, getClusterClient } = this.options; const clusterClient = await getClusterClient(); - const simulatedIndexMapping = await clusterClient.indices.simulateIndexTemplate({ name: index, }); @@ -180,35 +171,8 @@ export class ResourceInstaller { }); return; } catch (err) { - if (err.meta?.body?.error?.type !== 'illegal_argument_exception') { - /** - * We skip the rollover if we catch anything except for illegal_argument_exception - that's the error - * returned by ES when the mapping update contains a conflicting field definition (e.g., a field changes types). - * We expect to get that error for some mapping changes we might make, and in those cases, - * we want to continue to rollover the index. Other errors are unexpected. - */ - logger.error(`Failed to PUT mapping for alias ${alias}: ${err.message}`); - return; - } - const newIndexName = incrementIndexName(index); - if (newIndexName == null) { - logger.error(`Failed to increment write index name for alias: ${alias}`); - return; - } - try { - await clusterClient.indices.rollover({ - alias, - new_index: newIndexName, - }); - } catch (e) { - /** - * If we catch resource_already_exists_exception, that means that the index has been - * rolled over already — nothing to do for us in this case. - */ - if (e?.meta?.body?.error?.type !== 'resource_already_exists_exception') { - logger.error(`Failed to rollover index for alias ${alias}: ${e.message}`); - } - } + logger.error(`Failed to PUT mapping for alias ${alias}: ${err.message}`); + throw err; } } @@ -216,11 +180,12 @@ export class ResourceInstaller { // Namespace-level resources /** - * Installs resources tied to concrete namespace of an index: + * Installs and updates resources tied to concrete namespace of an index: * - namespaced index template + * - Index mappings for existing concrete indices * - concrete index (write target) if it doesn't exist */ - public async installNamespaceLevelResources( + public async installAndUpdateNamespaceLevelResources( indexInfo: IndexInfo, namespace: string ): Promise { @@ -230,15 +195,19 @@ export class ResourceInstaller { logger.info(`Installing namespace-level resources and creating concrete index for ${alias}`); + // Install / update the index template + await this.installNamespacedIndexTemplate(indexInfo, namespace); + // Update index mappings for indices matching this namespace. + await this.updateIndexMappings(indexInfo, namespace); + // If we find a concrete backing index which is the write index for the alias here, we shouldn't // be making a new concrete index. We return early because we don't need a new write target. const indexExists = await this.checkIfConcreteWriteIndexExists(indexInfo, namespace); if (indexExists) { return; + } else { + await this.createConcreteWriteIndex(indexInfo, namespace); } - - await this.installNamespacedIndexTemplate(indexInfo, namespace); - await this.createConcreteWriteIndex(indexInfo, namespace); } private async checkIfConcreteWriteIndexExists( diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts index c69677b091c9c..0617bc0a820ac 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts @@ -22,7 +22,6 @@ interface ConstructorOptions { logger: Logger; kibanaVersion: string; isWriteEnabled: boolean; - isIndexUpgradeEnabled: boolean; } /** @@ -44,7 +43,6 @@ export class RuleDataPluginService { getClusterClient: options.getClusterClient, logger: options.logger, isWriteEnabled: options.isWriteEnabled, - isIndexUpgradeEnabled: options.isIndexUpgradeEnabled, }); this.installCommonResources = Promise.resolve(right('ok')); @@ -154,6 +152,7 @@ export class RuleDataPluginService { isWriteEnabled: this.isWriteEnabled(), waitUntilReadyForReading, waitUntilReadyForWriting, + logger: this.options.logger, }); } From 8d1c96cd7efcdfa732413b88c65e8132b6e7682e Mon Sep 17 00:00:00 2001 From: Scotty Bollinger Date: Wed, 13 Oct 2021 13:44:50 -0500 Subject: [PATCH 18/35] [Workplace Search] Add Synchronize button to Source Overview page (#114842) * Add sync route * Add logic for triggering sync on server * Add button with confirm modal and description w/links --- .../applications/shared/constants/actions.ts | 5 ++ .../components/overview.test.tsx | 16 +++- .../content_sources/components/overview.tsx | 80 ++++++++++++++++++- .../views/content_sources/constants.ts | 43 ++++++++++ .../content_sources/source_logic.test.ts | 28 +++++++ .../views/content_sources/source_logic.ts | 11 +++ .../routes/workplace_search/sources.test.ts | 24 ++++++ .../server/routes/workplace_search/sources.ts | 20 +++++ 8 files changed, 222 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/constants/actions.ts b/x-pack/plugins/enterprise_search/public/applications/shared/constants/actions.ts index cb05311e11998..d43217fba1443 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/constants/actions.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/constants/actions.ts @@ -35,6 +35,11 @@ export const CANCEL_BUTTON_LABEL = i18n.translate( { defaultMessage: 'Cancel' } ); +export const START_BUTTON_LABEL = i18n.translate( + 'xpack.enterpriseSearch.actions.startButtonLabel', + { defaultMessage: 'Start' } +); + export const CONTINUE_BUTTON_LABEL = i18n.translate( 'xpack.enterpriseSearch.actions.continueButtonLabel', { defaultMessage: 'Continue' } diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.test.tsx index d99eac5de74e5..c9eb2e0afdf5e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.test.tsx @@ -5,20 +5,21 @@ * 2.0. */ -import { setMockValues } from '../../../../__mocks__/kea_logic'; +import { setMockValues, setMockActions } from '../../../../__mocks__/kea_logic'; import { fullContentSources } from '../../../__mocks__/content_sources.mock'; import React from 'react'; import { shallow } from 'enzyme'; -import { EuiEmptyPrompt, EuiPanel, EuiTable } from '@elastic/eui'; +import { EuiConfirmModal, EuiEmptyPrompt, EuiPanel, EuiTable } from '@elastic/eui'; import { ComponentLoader } from '../../../components/shared/component_loader'; import { Overview } from './overview'; describe('Overview', () => { + const initializeSourceSynchronization = jest.fn(); const contentSource = fullContentSources[0]; const dataLoading = false; const isOrganization = true; @@ -31,6 +32,7 @@ describe('Overview', () => { beforeEach(() => { setMockValues({ ...mockValues }); + setMockActions({ initializeSourceSynchronization }); }); it('renders', () => { @@ -118,4 +120,14 @@ describe('Overview', () => { expect(wrapper.find('[data-test-subj="DocumentPermissionsDisabled"]')).toHaveLength(1); }); + + it('handles confirmModal submission', () => { + const wrapper = shallow(); + const button = wrapper.find('[data-test-subj="SyncButton"]'); + button.prop('onClick')!({} as any); + const modal = wrapper.find(EuiConfirmModal); + modal.prop('onConfirm')!({} as any); + + expect(initializeSourceSynchronization).toHaveBeenCalled(); + }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx index a4fe0329e6c42..899d9dceebe3e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx @@ -5,11 +5,13 @@ * 2.0. */ -import React from 'react'; +import React, { useState } from 'react'; -import { useValues } from 'kea'; +import { useValues, useActions } from 'kea'; import { + EuiButton, + EuiConfirmModal, EuiEmptyPrompt, EuiFlexGroup, EuiFlexItem, @@ -30,7 +32,8 @@ import { } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiListGroupItemTo } from '../../../../shared/react_router_helpers'; +import { CANCEL_BUTTON_LABEL, START_BUTTON_LABEL } from '../../../../shared/constants'; +import { EuiListGroupItemTo, EuiLinkTo } from '../../../../shared/react_router_helpers'; import { AppLogic } from '../../../app_logic'; import aclImage from '../../../assets/supports_acl.svg'; import { ComponentLoader } from '../../../components/shared/component_loader'; @@ -48,7 +51,10 @@ import { DOCUMENT_PERMISSIONS_DOCS_URL, ENT_SEARCH_LICENSE_MANAGEMENT, EXTERNAL_IDENTITIES_DOCS_URL, + SYNC_FREQUENCY_PATH, + BLOCKED_TIME_WINDOWS_PATH, getGroupPath, + getContentSourcePath, } from '../../../routes'; import { SOURCES_NO_CONTENT_TITLE, @@ -77,6 +83,12 @@ import { LEARN_CUSTOM_FEATURES_BUTTON, DOC_PERMISSIONS_DESCRIPTION, CUSTOM_CALLOUT_TITLE, + SOURCE_SYNCHRONIZATION_TITLE, + SOURCE_SYNC_FREQUENCY_LINK_LABEL, + SOURCE_BLOCKED_TIME_WINDOWS_LINK_LABEL, + SOURCE_SYNCHRONIZATION_BUTTON_LABEL, + SOURCE_SYNC_CONFIRM_TITLE, + SOURCE_SYNC_CONFIRM_MESSAGE, } from '../constants'; import { SourceLogic } from '../source_logic'; @@ -84,6 +96,7 @@ import { SourceLayout } from './source_layout'; export const Overview: React.FC = () => { const { contentSource } = useValues(SourceLogic); + const { initializeSourceSynchronization } = useActions(SourceLogic); const { isOrganization } = useValues(AppLogic); const { @@ -99,8 +112,20 @@ export const Overview: React.FC = () => { indexPermissions, hasPermissions, isFederatedSource, + isIndexedSource, } = contentSource; + const [isSyncing, setIsSyncing] = useState(false); + const [isModalVisible, setIsModalVisible] = useState(false); + const closeModal = () => setIsModalVisible(false); + const handleSyncClick = () => setIsModalVisible(true); + + const onSyncConfirm = () => { + initializeSourceSynchronization(id); + setIsSyncing(true); + closeModal(); + }; + const DocumentSummary = () => { let totalDocuments = 0; const tableContent = summary?.map((item, index) => { @@ -451,9 +476,57 @@ export const Overview: React.FC = () => { ); + const syncTriggerCallout = ( + + + +

{SOURCE_SYNCHRONIZATION_TITLE}
+ + + + + {SOURCE_SYNCHRONIZATION_BUTTON_LABEL} + + + + + {SOURCE_SYNC_FREQUENCY_LINK_LABEL} + + ), + blockTimeWindowsLink: ( + + {SOURCE_BLOCKED_TIME_WINDOWS_LINK_LABEL} + + ), + }} + /> + + + + ); + + const syncConfirmModal = ( + +

{SOURCE_SYNC_CONFIRM_MESSAGE}

+ + ); + return ( + {isModalVisible && syncConfirmModal} @@ -513,6 +586,7 @@ export const Overview: React.FC = () => { )} )} + {isIndexedSource && isOrganization && syncTriggerCallout} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/constants.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/constants.ts index 14d0a7f196ae8..f44dbae0608ea 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/constants.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/constants.ts @@ -579,6 +579,49 @@ export const SOURCE_SYNCHRONIZATION_FREQUENCY_TITLE = i18n.translate( } ); +export const SOURCE_SYNCHRONIZATION_TITLE = i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.sources.sourceSynchronizationTitle', + { + defaultMessage: 'Synchronization', + } +); + +export const SOURCE_SYNCHRONIZATION_BUTTON_LABEL = i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.sources.sourceSynchronizationButtonLabel', + { + defaultMessage: 'Synchronize content', + } +); + +export const SOURCE_SYNC_FREQUENCY_LINK_LABEL = i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.sources.sourceSyncFrequencyLinkLabel', + { + defaultMessage: 'sync frequency', + } +); + +export const SOURCE_BLOCKED_TIME_WINDOWS_LINK_LABEL = i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.sources.sourceBlockedTimeWindowsLinkLabel', + { + defaultMessage: 'blocked time windows', + } +); + +export const SOURCE_SYNC_CONFIRM_TITLE = i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.sources.sourceSyncConfirmTitle', + { + defaultMessage: 'Start new content sync?', + } +); + +export const SOURCE_SYNC_CONFIRM_MESSAGE = i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.sources.sourceSyncConfirmMessage', + { + defaultMessage: + 'Are you sure you would like to continue with this request and stop all other syncs?', + } +); + export const SOURCE_SYNC_FREQUENCY_TITLE = i18n.translate( 'xpack.enterpriseSearch.workplaceSearch.sources.sourceSyncFrequencyTitle', { diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.test.ts index 1fb4477cea5c0..fb88360de5df0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.test.ts @@ -429,6 +429,34 @@ describe('SourceLogic', () => { }); }); + describe('initializeSourceSynchronization', () => { + it('calls API and fetches fresh source state', async () => { + const initializeSourceSpy = jest.spyOn(SourceLogic.actions, 'initializeSource'); + const promise = Promise.resolve(contentSource); + http.post.mockReturnValue(promise); + SourceLogic.actions.initializeSourceSynchronization(contentSource.id); + + expect(http.post).toHaveBeenCalledWith('/internal/workplace_search/org/sources/123/sync'); + await promise; + expect(initializeSourceSpy).toHaveBeenCalledWith(contentSource.id); + }); + + it('handles error', async () => { + const error = { + response: { + error: 'this is an error', + status: 400, + }, + }; + const promise = Promise.reject(error); + http.post.mockReturnValue(promise); + SourceLogic.actions.initializeSourceSynchronization(contentSource.id); + await expectedAsyncError(promise); + + expect(flashAPIErrors).toHaveBeenCalledWith(error); + }); + }); + it('resetSourceState', () => { SourceLogic.actions.resetSourceState(); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.ts index d10400bc5ba2d..9dcd0824cad11 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.ts @@ -27,6 +27,7 @@ export interface SourceActions { onUpdateSourceName(name: string): string; setSearchResults(searchResultsResponse: SearchResultsResponse): SearchResultsResponse; initializeFederatedSummary(sourceId: string): { sourceId: string }; + initializeSourceSynchronization(sourceId: string): { sourceId: string }; onUpdateSummary(summary: DocumentSummaryItem[]): DocumentSummaryItem[]; setContentFilterValue(contentFilterValue: string): string; setActivePage(activePage: number): number; @@ -81,6 +82,7 @@ export const SourceLogic = kea>({ setActivePage: (activePage: number) => activePage, initializeSource: (sourceId: string) => ({ sourceId }), initializeFederatedSummary: (sourceId: string) => ({ sourceId }), + initializeSourceSynchronization: (sourceId: string) => ({ sourceId }), searchContentSourceDocuments: (sourceId: string) => ({ sourceId }), updateContentSource: (sourceId: string, source: SourceUpdatePayload) => ({ sourceId, source }), removeContentSource: (sourceId: string) => ({ @@ -254,6 +256,15 @@ export const SourceLogic = kea>({ actions.setButtonNotLoading(); } }, + initializeSourceSynchronization: async ({ sourceId }) => { + const route = `/internal/workplace_search/org/sources/${sourceId}/sync`; + try { + await HttpLogic.values.http.post(route); + actions.initializeSource(sourceId); + } catch (e) { + flashAPIErrors(e); + } + }, onUpdateSourceName: (name: string) => { flashSuccessToast( i18n.translate( diff --git a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts index 706bc8a4853a7..961635c3f9001 100644 --- a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts @@ -42,6 +42,7 @@ import { registerOrgSourceDownloadDiagnosticsRoute, registerOrgSourceOauthConfigurationsRoute, registerOrgSourceOauthConfigurationRoute, + registerOrgSourceSynchronizeRoute, registerOauthConnectorParamsRoute, } from './sources'; @@ -1252,6 +1253,29 @@ describe('sources routes', () => { }); }); + describe('POST /internal/workplace_search/org/sources/{id}/sync', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ + method: 'post', + path: '/internal/workplace_search/org/sources/{id}/sync', + }); + + registerOrgSourceSynchronizeRoute({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request handler', () => { + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/ws/org/sources/:id/sync', + }); + }); + }); + describe('GET /internal/workplace_search/sources/create', () => { const tokenPackage = 'some_encrypted_secrets'; diff --git a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts index 660294a5e1ddd..011fe341d6edf 100644 --- a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts +++ b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts @@ -891,6 +891,25 @@ export function registerOrgSourceOauthConfigurationRoute({ ); } +export function registerOrgSourceSynchronizeRoute({ + router, + enterpriseSearchRequestHandler, +}: RouteDependencies) { + router.post( + { + path: '/internal/workplace_search/org/sources/{id}/sync', + validate: { + params: schema.object({ + id: schema.string(), + }), + }, + }, + enterpriseSearchRequestHandler.createRequest({ + path: '/ws/org/sources/:id/sync', + }) + ); +} + // Same route is used for org and account. `state` passes the context. export function registerOauthConnectorParamsRoute({ router, @@ -956,5 +975,6 @@ export const registerSourcesRoutes = (dependencies: RouteDependencies) => { registerOrgSourceDownloadDiagnosticsRoute(dependencies); registerOrgSourceOauthConfigurationsRoute(dependencies); registerOrgSourceOauthConfigurationRoute(dependencies); + registerOrgSourceSynchronizeRoute(dependencies); registerOauthConnectorParamsRoute(dependencies); }; From a532ea5c05d41e2d0fd3a8dee66316e594c8d698 Mon Sep 17 00:00:00 2001 From: Davey Holler Date: Wed, 13 Oct 2021 12:21:46 -0700 Subject: [PATCH 19/35] [App Search] Static Curations History Tab (#113481) --- .../components/curations/curations_logic.ts | 2 +- .../curations/views/curations.test.tsx | 18 ++++++- .../components/curations/views/curations.tsx | 12 +++++ .../curation_changes_panel.test.tsx | 22 ++++++++ .../components/curation_changes_panel.tsx | 23 ++++++++ .../ignored_suggestions_panel.test.tsx | 25 +++++++++ .../components/ignored_suggestions_panel.tsx | 53 +++++++++++++++++++ .../curations_history/components/index.ts | 10 ++++ .../rejected_curations_panel.test.tsx | 22 ++++++++ .../components/rejected_curations_panel.tsx | 23 ++++++++ .../curations_history.test.tsx | 27 ++++++++++ .../curations_history/curations_history.tsx | 36 +++++++++++++ .../views/curations_history/index.ts | 8 +++ 13 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/curation_changes_panel.test.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/curation_changes_panel.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/ignored_suggestions_panel.test.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/ignored_suggestions_panel.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/index.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/rejected_curations_panel.test.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/rejected_curations_panel.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/curations_history.test.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/curations_history.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/index.ts diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.ts index e623379e58d3f..04d04b297050a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.ts @@ -23,7 +23,7 @@ import { EngineLogic, generateEnginePath } from '../engine'; import { DELETE_CONFIRMATION_MESSAGE, DELETE_SUCCESS_MESSAGE } from './constants'; import { Curation, CurationsAPIResponse } from './types'; -type CurationsPageTabs = 'overview' | 'settings'; +type CurationsPageTabs = 'overview' | 'settings' | 'history'; interface CurationsValues { dataLoading: boolean; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.test.tsx index 7e357cae4343c..f7e9f5437fc3f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.test.tsx @@ -19,6 +19,7 @@ import { EuiTab } from '@elastic/eui'; import { getPageHeaderTabs, getPageTitle } from '../../../../test_helpers'; import { Curations } from './curations'; +import { CurationsHistory } from './curations_history/curations_history'; import { CurationsOverview } from './curations_overview'; import { CurationsSettings } from './curations_settings'; @@ -70,7 +71,10 @@ describe('Curations', () => { expect(actions.onSelectPageTab).toHaveBeenNthCalledWith(1, 'overview'); tabs.at(1).simulate('click'); - expect(actions.onSelectPageTab).toHaveBeenNthCalledWith(2, 'settings'); + expect(actions.onSelectPageTab).toHaveBeenNthCalledWith(2, 'history'); + + tabs.at(2).simulate('click'); + expect(actions.onSelectPageTab).toHaveBeenNthCalledWith(3, 'settings'); }); it('renders an overview view', () => { @@ -83,12 +87,22 @@ describe('Curations', () => { expect(wrapper.find(CurationsOverview)).toHaveLength(1); }); + it('renders a history view', () => { + setMockValues({ ...values, selectedPageTab: 'history' }); + const wrapper = shallow(); + const tabs = getPageHeaderTabs(wrapper).find(EuiTab); + + expect(tabs.at(1).prop('isSelected')).toEqual(true); + + expect(wrapper.find(CurationsHistory)).toHaveLength(1); + }); + it('renders a settings view', () => { setMockValues({ ...values, selectedPageTab: 'settings' }); const wrapper = shallow(); const tabs = getPageHeaderTabs(wrapper).find(EuiTab); - expect(tabs.at(1).prop('isSelected')).toEqual(true); + expect(tabs.at(2).prop('isSelected')).toEqual(true); expect(wrapper.find(CurationsSettings)).toHaveLength(1); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx index 9584b21424fe3..c55fde7626488 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx @@ -21,6 +21,7 @@ import { CURATIONS_OVERVIEW_TITLE, CREATE_NEW_CURATION_TITLE } from '../constant import { CurationsLogic } from '../curations_logic'; import { getCurationsBreadcrumbs } from '../utils'; +import { CurationsHistory } from './curations_history/curations_history'; import { CurationsOverview } from './curations_overview'; import { CurationsSettings } from './curations_settings'; @@ -39,6 +40,16 @@ export const Curations: React.FC = () => { isSelected: selectedPageTab === 'overview', onClick: () => onSelectPageTab('overview'), }, + { + label: i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.curations.historyPageTabLabel', + { + defaultMessage: 'History', + } + ), + isSelected: selectedPageTab === 'history', + onClick: () => onSelectPageTab('history'), + }, { label: i18n.translate( 'xpack.enterpriseSearch.appSearch.engine.curations.settingsPageTabLabel', @@ -74,6 +85,7 @@ export const Curations: React.FC = () => { isLoading={dataLoading && !curations.length} > {selectedPageTab === 'overview' && } + {selectedPageTab === 'history' && } {selectedPageTab === 'settings' && } ); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/curation_changes_panel.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/curation_changes_panel.test.tsx new file mode 100644 index 0000000000000..7fc06beaa86a9 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/curation_changes_panel.test.tsx @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { DataPanel } from '../../../../data_panel'; + +import { CurationChangesPanel } from './curation_changes_panel'; + +describe('CurationChangesPanel', () => { + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.is(DataPanel)).toBe(true); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/curation_changes_panel.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/curation_changes_panel.tsx new file mode 100644 index 0000000000000..0aaf20485966e --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/curation_changes_panel.tsx @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { DataPanel } from '../../../../data_panel'; + +export const CurationChangesPanel: React.FC = () => { + return ( + Automated curation changes} + subtitle={A detailed log of recent changes to your automated curations} + iconType="visTable" + hasBorder + > + Embedded logs view goes here... + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/ignored_suggestions_panel.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/ignored_suggestions_panel.test.tsx new file mode 100644 index 0000000000000..b09981748f19c --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/ignored_suggestions_panel.test.tsx @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { EuiBasicTable } from '@elastic/eui'; + +import { DataPanel } from '../../../../data_panel'; + +import { IgnoredSuggestionsPanel } from './ignored_suggestions_panel'; + +describe('IgnoredSuggestionsPanel', () => { + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.is(DataPanel)).toBe(true); + expect(wrapper.find(EuiBasicTable)).toHaveLength(1); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/ignored_suggestions_panel.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/ignored_suggestions_panel.tsx new file mode 100644 index 0000000000000..f2fdfd55a7e5a --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/ignored_suggestions_panel.tsx @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { CustomItemAction, EuiBasicTable, EuiBasicTableColumn, EuiLink } from '@elastic/eui'; + +import { DataPanel } from '../../../../data_panel'; +import { CurationSuggestion } from '../../../types'; + +export const IgnoredSuggestionsPanel: React.FC = () => { + const ignoredSuggestions: CurationSuggestion[] = []; + + const allowSuggestion = (query: string) => alert(query); + + const actions: Array> = [ + { + render: (item: CurationSuggestion) => { + return ( + allowSuggestion(item.query)} color="primary"> + Allow + + ); + }, + }, + ]; + + const columns: Array> = [ + { + field: 'query', + name: 'Query', + sortable: true, + }, + { + actions, + }, + ]; + + return ( + Ignored queries} + subtitle={You won’t be notified about suggestions for these queries} + iconType="eyeClosed" + hasBorder + > + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/index.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/index.ts new file mode 100644 index 0000000000000..2e16d9bde8550 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { CurationChangesPanel } from './curation_changes_panel'; +export { IgnoredSuggestionsPanel } from './ignored_suggestions_panel'; +export { RejectedCurationsPanel } from './rejected_curations_panel'; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/rejected_curations_panel.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/rejected_curations_panel.test.tsx new file mode 100644 index 0000000000000..a40eb8895ad69 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/rejected_curations_panel.test.tsx @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { DataPanel } from '../../../../data_panel'; + +import { RejectedCurationsPanel } from './rejected_curations_panel'; + +describe('RejectedCurationsPanel', () => { + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.is(DataPanel)).toBe(true); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/rejected_curations_panel.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/rejected_curations_panel.tsx new file mode 100644 index 0000000000000..51719b4eebbd7 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/components/rejected_curations_panel.tsx @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { DataPanel } from '../../../../data_panel'; + +export const RejectedCurationsPanel: React.FC = () => { + return ( + Rececntly rejected sugggestions} + subtitle={Recent suggestions that are still valid can be re-enabled from here} + iconType="crossInACircleFilled" + hasBorder + > + Embedded logs view goes here... + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/curations_history.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/curations_history.test.tsx new file mode 100644 index 0000000000000..1ebd4da694d54 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/curations_history.test.tsx @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { + CurationChangesPanel, + IgnoredSuggestionsPanel, + RejectedCurationsPanel, +} from './components'; +import { CurationsHistory } from './curations_history'; + +describe('CurationsHistory', () => { + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(CurationChangesPanel)).toHaveLength(1); + expect(wrapper.find(RejectedCurationsPanel)).toHaveLength(1); + expect(wrapper.find(IgnoredSuggestionsPanel)).toHaveLength(1); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/curations_history.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/curations_history.tsx new file mode 100644 index 0000000000000..6db62820b1cdb --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/curations_history.tsx @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; + +import { + CurationChangesPanel, + IgnoredSuggestionsPanel, + RejectedCurationsPanel, +} from './components'; + +export const CurationsHistory: React.FC = () => { + return ( + + + + + + + + + + + + + + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/index.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/index.ts new file mode 100644 index 0000000000000..bddc156f7920e --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_history/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { CurationsHistory } from './curations_history'; From fdd72a9e80621d31f562ddae4413967ca375031f Mon Sep 17 00:00:00 2001 From: Orhan Toy Date: Wed, 13 Oct 2021 21:36:03 +0200 Subject: [PATCH 20/35] [App Search] [Crawler] Add tooltip to explain path pattern (#114779) 7.13.0 adds a wildcard character to (non-regex) path patterns. This change updates the UI help text to explain this. --- .../crawler/components/crawl_rules_table.tsx | 37 +++++++++++++++---- .../components/crawler/utils.test.ts | 26 +++++++++++++ .../app_search/components/crawler/utils.ts | 22 +++++++++++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/crawl_rules_table.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/crawl_rules_table.tsx index d8d5d9d10b3b7..df7ea80779acf 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/crawl_rules_table.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/crawl_rules_table.tsx @@ -9,7 +9,16 @@ import React from 'react'; import { useActions } from 'kea'; -import { EuiCode, EuiFieldText, EuiLink, EuiSelect, EuiText } from '@elastic/eui'; +import { + EuiCode, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiIconTip, + EuiLink, + EuiSelect, + EuiText, +} from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -27,6 +36,7 @@ import { getReadableCrawlerPolicy, getReadableCrawlerRule, } from '../types'; +import { getCrawlRulePathPatternTooltip } from '../utils'; interface CrawlRulesTableProps { description?: React.ReactNode; @@ -130,13 +140,24 @@ export const CrawlRulesTable: React.FC = ({ }, { editingRender: (crawlRule, onChange, { isInvalid, isLoading }) => ( - onChange(e.target.value)} - disabled={isLoading} - isInvalid={isInvalid} - /> + + + onChange(e.target.value)} + disabled={isLoading} + isInvalid={isInvalid} + /> + + + + + ), render: (crawlRule) => {(crawlRule as CrawlRule).pattern}, name: i18n.translate( diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.test.ts index fc810ba8fd7cb..0fc608ac6f5e4 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.test.ts @@ -26,6 +26,7 @@ import { crawlRequestServerToClient, getDeleteDomainConfirmationMessage, getDeleteDomainSuccessMessage, + getCrawlRulePathPatternTooltip, } from './utils'; const DEFAULT_CRAWL_RULE: CrawlRule = { @@ -292,3 +293,28 @@ describe('getDeleteDomainSuccessMessage', () => { expect(getDeleteDomainSuccessMessage('https://elastic.co/')).toContain('https://elastic.co'); }); }); + +describe('getCrawlRulePathPatternTooltip', () => { + it('includes regular expression', () => { + const crawlRule: CrawlRule = { + id: '-', + policy: CrawlerPolicies.allow, + rule: CrawlerRules.regex, + pattern: '.*', + }; + + expect(getCrawlRulePathPatternTooltip(crawlRule)).toContain('regular expression'); + }); + + it('includes meta', () => { + const crawlRule: CrawlRule = { + id: '-', + policy: CrawlerPolicies.allow, + rule: CrawlerRules.beginsWith, + pattern: '/elastic', + }; + + expect(getCrawlRulePathPatternTooltip(crawlRule)).not.toContain('regular expression'); + expect(getCrawlRulePathPatternTooltip(crawlRule)).toContain('meta'); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.ts index 9c94040355d47..817f10b70dca5 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.ts @@ -16,6 +16,8 @@ import { CrawlerDomainValidationStep, CrawlRequestFromServer, CrawlRequest, + CrawlRule, + CrawlerRules, CrawlEventFromServer, CrawlEvent, } from './types'; @@ -159,3 +161,23 @@ export const getDeleteDomainSuccessMessage = (domainUrl: string) => { } ); }; + +export const getCrawlRulePathPatternTooltip = (crawlRule: CrawlRule) => { + if (crawlRule.rule === CrawlerRules.regex) { + return i18n.translate( + 'xpack.enterpriseSearch.appSearch.crawler.crawlRulesTable.regexPathPatternTooltip', + { + defaultMessage: + 'The path pattern is a regular expression compatible with the Ruby language regular expression engine.', + } + ); + } + + return i18n.translate( + 'xpack.enterpriseSearch.appSearch.crawler.crawlRulesTable.pathPatternTooltip', + { + defaultMessage: + 'The path pattern is a literal string except for the asterisk (*) character, which is a meta character that will match anything.', + } + ); +}; From 491fcd5c36af3304cc4023e889dc150137acb1a9 Mon Sep 17 00:00:00 2001 From: Kevin Lacabane Date: Wed, 13 Oct 2021 21:45:49 +0200 Subject: [PATCH 21/35] [Stack Monitoring] fix beats pages test-subj attributes (#114835) * fix beats pages test-subj attributes * fix eslint errors --- .../public/application/pages/beats/beats_template.tsx | 2 ++ .../public/application/pages/beats/instances.tsx | 7 +------ .../public/application/pages/beats/overview.tsx | 9 ++------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/monitoring/public/application/pages/beats/beats_template.tsx b/x-pack/plugins/monitoring/public/application/pages/beats/beats_template.tsx index 3bab01af8ceb7..7a070c735bbea 100644 --- a/x-pack/plugins/monitoring/public/application/pages/beats/beats_template.tsx +++ b/x-pack/plugins/monitoring/public/application/pages/beats/beats_template.tsx @@ -23,6 +23,7 @@ export const BeatsTemplate: React.FC = ({ instance, ...props defaultMessage: 'Overview', }), route: '/beats', + testSubj: 'beatsOverviewPage', }); tabs.push({ id: 'instances', @@ -30,6 +31,7 @@ export const BeatsTemplate: React.FC = ({ instance, ...props defaultMessage: 'Instances', }), route: '/beats/beats', + testSubj: 'beatsListingPage', }); } else { tabs.push({ diff --git a/x-pack/plugins/monitoring/public/application/pages/beats/instances.tsx b/x-pack/plugins/monitoring/public/application/pages/beats/instances.tsx index 489ad110c40fd..4611f17159621 100644 --- a/x-pack/plugins/monitoring/public/application/pages/beats/instances.tsx +++ b/x-pack/plugins/monitoring/public/application/pages/beats/instances.tsx @@ -71,12 +71,7 @@ export const BeatsInstancesPage: React.FC = ({ clusters }) => { ]); return ( - +
= ({ clusters }) => { }; return ( - -
{renderOverview(data)}
+ +
{renderOverview(data)}
 ); }; From e1e1830f15f96ced2deacd614663c17ab4327890 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 13 Oct 2021 14:20:56 -0600 Subject: [PATCH 22/35] [Breaking] Remove `/api/settings` & the `xpack_legacy` plugin. (#114730) --- .github/CODEOWNERS | 1 - api_docs/plugin_directory.mdx | 1 - docs/developer/plugin-list.asciidoc | 4 - x-pack/plugins/xpack_legacy/README.md | 3 - x-pack/plugins/xpack_legacy/jest.config.js | 15 --- x-pack/plugins/xpack_legacy/kibana.json | 12 -- x-pack/plugins/xpack_legacy/server/index.ts | 12 -- x-pack/plugins/xpack_legacy/server/plugin.ts | 45 ------- .../server/routes/settings.test.ts | 115 ------------------ .../xpack_legacy/server/routes/settings.ts | 96 --------------- x-pack/plugins/xpack_legacy/tsconfig.json | 17 --- x-pack/test/api_integration/apis/index.ts | 1 - .../apis/xpack_legacy/index.js | 12 -- .../apis/xpack_legacy/settings/index.js | 12 -- .../apis/xpack_legacy/settings/settings.js | 40 ------ x-pack/test/tsconfig.json | 3 +- 16 files changed, 1 insertion(+), 388 deletions(-) delete mode 100644 x-pack/plugins/xpack_legacy/README.md delete mode 100644 x-pack/plugins/xpack_legacy/jest.config.js delete mode 100644 x-pack/plugins/xpack_legacy/kibana.json delete mode 100644 x-pack/plugins/xpack_legacy/server/index.ts delete mode 100644 x-pack/plugins/xpack_legacy/server/plugin.ts delete mode 100644 x-pack/plugins/xpack_legacy/server/routes/settings.test.ts delete mode 100644 x-pack/plugins/xpack_legacy/server/routes/settings.ts delete mode 100644 x-pack/plugins/xpack_legacy/tsconfig.json delete mode 100644 x-pack/test/api_integration/apis/xpack_legacy/index.js delete mode 100644 x-pack/test/api_integration/apis/xpack_legacy/settings/index.js delete mode 100644 x-pack/test/api_integration/apis/xpack_legacy/settings/settings.js diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 91a8f8c2d5998..1e0a8b187c778 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -254,7 +254,6 @@ /src/plugins/kibana_overview/ @elastic/kibana-core /x-pack/plugins/global_search_bar/ @elastic/kibana-core #CC# /src/core/server/csp/ @elastic/kibana-core -#CC# /src/plugins/xpack_legacy/ @elastic/kibana-core #CC# /src/plugins/saved_objects/ @elastic/kibana-core #CC# /x-pack/plugins/cloud/ @elastic/kibana-core #CC# /x-pack/plugins/features/ @elastic/kibana-core diff --git a/api_docs/plugin_directory.mdx b/api_docs/plugin_directory.mdx index 533a86b9e806f..6380831a8c6c3 100644 --- a/api_docs/plugin_directory.mdx +++ b/api_docs/plugin_directory.mdx @@ -153,7 +153,6 @@ warning: This document is auto-generated and is meant to be viewed inside our ex | | [Vis Editors](https://github.com/orgs/elastic/teams/kibana-vis-editors) | Contains the shared architecture among all the legacy visualizations, e.g. the visualization type registry or the visualization embeddable. | 304 | 13 | 286 | 16 | | | [Vis Editors](https://github.com/orgs/elastic/teams/kibana-vis-editors) | Contains the visualize application which includes the listing page and the app frame, which will load the visualization's editor. | 24 | 0 | 23 | 1 | | watcher | [Stack Management](https://github.com/orgs/elastic/teams/kibana-stack-management) | - | 0 | 0 | 0 | 0 | -| xpackLegacy | [Kibana Core](https://github.com/orgs/elastic/teams/kibana-core) | - | 0 | 0 | 0 | 0 | ## Package Directory diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 9581848be9e53..3d1fcd51837a3 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -617,10 +617,6 @@ in their infrastructure. |This plugins adopts some conventions in addition to or in place of conventions in Kibana (at the time of the plugin's creation): -|{kib-repo}blob/{branch}/x-pack/plugins/xpack_legacy/README.md[xpackLegacy] -|Contains HTTP endpoints and UiSettings that are slated for removal. - - |=== include::{kibana-root}/src/plugins/dashboard/README.asciidoc[leveloffset=+1] diff --git a/x-pack/plugins/xpack_legacy/README.md b/x-pack/plugins/xpack_legacy/README.md deleted file mode 100644 index be43825347959..0000000000000 --- a/x-pack/plugins/xpack_legacy/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Xpack Leagcy - -Contains HTTP endpoints and UiSettings that are slated for removal. diff --git a/x-pack/plugins/xpack_legacy/jest.config.js b/x-pack/plugins/xpack_legacy/jest.config.js deleted file mode 100644 index 5ad0fa36264d1..0000000000000 --- a/x-pack/plugins/xpack_legacy/jest.config.js +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -module.exports = { - preset: '@kbn/test', - rootDir: '../../..', - roots: ['/x-pack/plugins/xpack_legacy'], - coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/xpack_legacy', - coverageReporters: ['text', 'html'], - collectCoverageFrom: ['/x-pack/plugins/xpack_legacy/server/**/*.{ts,tsx}'], -}; diff --git a/x-pack/plugins/xpack_legacy/kibana.json b/x-pack/plugins/xpack_legacy/kibana.json deleted file mode 100644 index 9dd0ac8340183..0000000000000 --- a/x-pack/plugins/xpack_legacy/kibana.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "id": "xpackLegacy", - "owner": { - "name": "Kibana Core", - "githubTeam": "kibana-core" - }, - "version": "8.0.0", - "kibanaVersion": "kibana", - "server": true, - "ui": false, - "requiredPlugins": ["usageCollection"] -} diff --git a/x-pack/plugins/xpack_legacy/server/index.ts b/x-pack/plugins/xpack_legacy/server/index.ts deleted file mode 100644 index ee51afcca429f..0000000000000 --- a/x-pack/plugins/xpack_legacy/server/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { PluginInitializerContext } from '../../../../src/core/server'; -import { XpackLegacyPlugin } from './plugin'; - -export const plugin = (initializerContext: PluginInitializerContext) => - new XpackLegacyPlugin(initializerContext); diff --git a/x-pack/plugins/xpack_legacy/server/plugin.ts b/x-pack/plugins/xpack_legacy/server/plugin.ts deleted file mode 100644 index ffef7117bbbd8..0000000000000 --- a/x-pack/plugins/xpack_legacy/server/plugin.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - CoreStart, - CoreSetup, - Plugin, - PluginInitializerContext, -} from '../../../../src/core/server'; -import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; -import { registerSettingsRoute } from './routes/settings'; - -interface SetupPluginDeps { - usageCollection: UsageCollectionSetup; -} - -export class XpackLegacyPlugin implements Plugin { - constructor(private readonly initContext: PluginInitializerContext) {} - - public setup(core: CoreSetup, { usageCollection }: SetupPluginDeps) { - const router = core.http.createRouter(); - const globalConfig = this.initContext.config.legacy.get(); - const serverInfo = core.http.getServerInfo(); - - registerSettingsRoute({ - router, - usageCollection, - overallStatus$: core.status.overall$, - config: { - kibanaIndex: globalConfig.kibana.index, - kibanaVersion: this.initContext.env.packageInfo.version, - uuid: this.initContext.env.instanceUuid, - server: serverInfo, - }, - }); - } - - public start(core: CoreStart) {} - - public stop() {} -} diff --git a/x-pack/plugins/xpack_legacy/server/routes/settings.test.ts b/x-pack/plugins/xpack_legacy/server/routes/settings.test.ts deleted file mode 100644 index f265ea6ab125a..0000000000000 --- a/x-pack/plugins/xpack_legacy/server/routes/settings.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { BehaviorSubject } from 'rxjs'; -import { UnwrapPromise } from '@kbn/utility-types'; -import supertest from 'supertest'; - -import { ServiceStatus, ServiceStatusLevels } from '../../../../../src/core/server'; -import { - contextServiceMock, - elasticsearchServiceMock, - savedObjectsServiceMock, - executionContextServiceMock, -} from '../../../../../src/core/server/mocks'; -import { createHttpServer } from '../../../../../src/core/server/test_utils'; -import { registerSettingsRoute } from './settings'; - -type HttpService = ReturnType; -type HttpSetup = UnwrapPromise>; - -export function mockGetClusterInfo(clusterInfo: any) { - const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; - // @ts-ignore we only care about the response body - esClient.info.mockResolvedValue({ body: { ...clusterInfo } }); - return esClient; -} - -describe('/api/settings', () => { - let server: HttpService; - let httpSetup: HttpSetup; - let overallStatus$: BehaviorSubject; - - beforeEach(async () => { - server = createHttpServer(); - await server.preboot({ context: contextServiceMock.createPrebootContract() }); - httpSetup = await server.setup({ - context: contextServiceMock.createSetupContract({ - core: { - elasticsearch: { - client: { - asCurrentUser: mockGetClusterInfo({ cluster_uuid: 'yyy-yyyyy' }), - }, - }, - savedObjects: { - client: savedObjectsServiceMock.create(), - }, - }, - }), - executionContext: executionContextServiceMock.createInternalSetupContract(), - }); - - overallStatus$ = new BehaviorSubject({ - level: ServiceStatusLevels.available, - summary: 'everything is working', - }); - - const usageCollection = { - getCollectorByType: jest.fn().mockReturnValue({ - fetch: jest - .fn() - .mockReturnValue({ xpack: { default_admin_email: 'kibana-machine@elastic.co' } }), - }), - } as any; - - const router = httpSetup.createRouter(''); - registerSettingsRoute({ - router, - overallStatus$, - usageCollection, - config: { - kibanaIndex: '.kibana-test', - kibanaVersion: '8.8.8-SNAPSHOT', - server: { - name: 'mykibana', - hostname: 'mykibana.com', - port: 1234, - }, - uuid: 'xxx-xxxxx', - }, - }); - - await server.start(); - }); - - afterEach(async () => { - await server.stop(); - }); - - it('successfully returns data', async () => { - const response = await supertest(httpSetup.server.listener).get('/api/settings').expect(200); - expect(response.body).toMatchObject({ - cluster_uuid: 'yyy-yyyyy', - settings: { - xpack: { - default_admin_email: 'kibana-machine@elastic.co', - }, - kibana: { - uuid: 'xxx-xxxxx', - name: 'mykibana', - index: '.kibana-test', - host: 'mykibana.com', - locale: 'en', - transport_address: `mykibana.com:1234`, - version: '8.8.8', - snapshot: true, - status: 'green', - }, - }, - }); - }); -}); diff --git a/x-pack/plugins/xpack_legacy/server/routes/settings.ts b/x-pack/plugins/xpack_legacy/server/routes/settings.ts deleted file mode 100644 index b9052ca0c84e3..0000000000000 --- a/x-pack/plugins/xpack_legacy/server/routes/settings.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { i18n } from '@kbn/i18n'; -import { Observable } from 'rxjs'; -import { first } from 'rxjs/operators'; - -import { IRouter, ServiceStatus, ServiceStatusLevels } from '../../../../../src/core/server'; -import { UsageCollectionSetup } from '../../../../../src/plugins/usage_collection/server'; -import { KIBANA_SETTINGS_TYPE } from '../../../monitoring/common/constants'; -import { KibanaSettingsCollector } from '../../../monitoring/server'; - -const SNAPSHOT_REGEX = /-snapshot/i; - -export function registerSettingsRoute({ - router, - usageCollection, - overallStatus$, - config, -}: { - router: IRouter; - usageCollection: UsageCollectionSetup; - overallStatus$: Observable; - config: { - kibanaIndex: string; - kibanaVersion: string; - uuid: string; - server: { - name: string; - hostname: string; - port: number; - }; - }; -}) { - router.get( - { - path: '/api/settings', - validate: false, - }, - async (context, req, res) => { - const collectorFetchContext = { - esClient: context.core.elasticsearch.client.asCurrentUser, - soClient: context.core.savedObjects.client, - }; - - const settingsCollector = usageCollection.getCollectorByType(KIBANA_SETTINGS_TYPE) as - | KibanaSettingsCollector - | undefined; - if (!settingsCollector) { - throw new Error('The settings collector is not registered'); - } - - const settings = - (await settingsCollector.fetch(collectorFetchContext)) ?? - settingsCollector.getEmailValueStructure(null); - - const { body } = await collectorFetchContext.esClient.info({ filter_path: 'cluster_uuid' }); - const uuid: string = body.cluster_uuid; - - const overallStatus = await overallStatus$.pipe(first()).toPromise(); - - const kibana = { - uuid: config.uuid, - name: config.server.name, - index: config.kibanaIndex, - host: config.server.hostname, - port: config.server.port, - locale: i18n.getLocale(), - transport_address: `${config.server.hostname}:${config.server.port}`, - version: config.kibanaVersion.replace(SNAPSHOT_REGEX, ''), - snapshot: SNAPSHOT_REGEX.test(config.kibanaVersion), - status: ServiceStatusToLegacyState[overallStatus.level.toString()], - }; - return res.ok({ - body: { - cluster_uuid: uuid, - settings: { - ...settings, - kibana, - }, - }, - }); - } - ); -} - -const ServiceStatusToLegacyState: Record = { - [ServiceStatusLevels.critical.toString()]: 'red', - [ServiceStatusLevels.unavailable.toString()]: 'red', - [ServiceStatusLevels.degraded.toString()]: 'yellow', - [ServiceStatusLevels.available.toString()]: 'green', -}; diff --git a/x-pack/plugins/xpack_legacy/tsconfig.json b/x-pack/plugins/xpack_legacy/tsconfig.json deleted file mode 100644 index 57fccc031a0cf..0000000000000 --- a/x-pack/plugins/xpack_legacy/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "extends": "../../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./target/types", - "emitDeclarationOnly": true, - "declaration": true, - "declarationMap": true - }, - "include": [ - "server/**/*", - ], - "references": [ - { "path": "../../../src/core/tsconfig.json" }, - { "path": "../../../src/plugins/usage_collection/tsconfig.json" }, - { "path": "../monitoring/tsconfig.json" }, - ] -} diff --git a/x-pack/test/api_integration/apis/index.ts b/x-pack/test/api_integration/apis/index.ts index eed39fd6dc6dc..c3d08ba306692 100644 --- a/x-pack/test/api_integration/apis/index.ts +++ b/x-pack/test/api_integration/apis/index.ts @@ -16,7 +16,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./security')); loadTestFile(require.resolve('./spaces')); loadTestFile(require.resolve('./monitoring')); - loadTestFile(require.resolve('./xpack_legacy')); loadTestFile(require.resolve('./features')); loadTestFile(require.resolve('./telemetry')); loadTestFile(require.resolve('./logstash')); diff --git a/x-pack/test/api_integration/apis/xpack_legacy/index.js b/x-pack/test/api_integration/apis/xpack_legacy/index.js deleted file mode 100644 index 4d3046286fb9d..0000000000000 --- a/x-pack/test/api_integration/apis/xpack_legacy/index.js +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export default function ({ loadTestFile }) { - describe('xpack_legacy', () => { - loadTestFile(require.resolve('./settings')); - }); -} diff --git a/x-pack/test/api_integration/apis/xpack_legacy/settings/index.js b/x-pack/test/api_integration/apis/xpack_legacy/settings/index.js deleted file mode 100644 index 3d8ce8d33b3d4..0000000000000 --- a/x-pack/test/api_integration/apis/xpack_legacy/settings/index.js +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export default function ({ loadTestFile }) { - describe('Settings', () => { - loadTestFile(require.resolve('./settings')); - }); -} diff --git a/x-pack/test/api_integration/apis/xpack_legacy/settings/settings.js b/x-pack/test/api_integration/apis/xpack_legacy/settings/settings.js deleted file mode 100644 index 6a82c5468a2c4..0000000000000 --- a/x-pack/test/api_integration/apis/xpack_legacy/settings/settings.js +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; - -export default function ({ getService }) { - const supertest = getService('supertest'); - const esArchiver = getService('esArchiver'); - - describe('/api/settings', () => { - describe('with trial license clusters', () => { - const archive = 'x-pack/test/functional/es_archives/monitoring/multicluster'; - - before('load clusters archive', () => { - return esArchiver.load(archive); - }); - - after('unload clusters archive', () => { - return esArchiver.unload(archive); - }); - - it('should load multiple clusters', async () => { - const { body } = await supertest.get('/api/settings').set('kbn-xsrf', 'xxx').expect(200); - expect(body.cluster_uuid.length > 1).to.eql(true); - expect(body.settings.kibana.uuid.length > 0).to.eql(true); - expect(body.settings.kibana.name.length > 0).to.eql(true); - expect(body.settings.kibana.index.length > 0).to.eql(true); - expect(body.settings.kibana.host.length > 0).to.eql(true); - expect(body.settings.kibana.transport_address.length > 0).to.eql(true); - expect(body.settings.kibana.version.length > 0).to.eql(true); - expect(body.settings.kibana.status.length > 0).to.eql(true); - expect(body.settings.xpack.default_admin_email).to.eql(null); - }); - }); - }); -} diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 173403743235b..1ffe3834d782d 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -102,7 +102,6 @@ { "path": "../plugins/remote_clusters/tsconfig.json" }, { "path": "../plugins/cross_cluster_replication/tsconfig.json" }, { "path": "../plugins/index_lifecycle_management/tsconfig.json"}, - { "path": "../plugins/uptime/tsconfig.json" }, - { "path": "../plugins/xpack_legacy/tsconfig.json" } + { "path": "../plugins/uptime/tsconfig.json" } ] } From fda421fab61bba5799ddcff1d70c8f4cc291a695 Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Wed, 13 Oct 2021 16:34:31 -0400 Subject: [PATCH 23/35] Always call resolve (#114670) --- .../components/alert_details_route.test.tsx | 118 +++--------------- .../components/alert_details_route.tsx | 24 +--- 2 files changed, 26 insertions(+), 116 deletions(-) diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details_route.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details_route.test.tsx index c07138990f88d..847c6c65464b2 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details_route.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details_route.test.tsx @@ -19,20 +19,6 @@ import { spacesPluginMock } from '../../../../../../spaces/public/mocks'; import { useKibana } from '../../../../common/lib/kibana'; jest.mock('../../../../common/lib/kibana'); -class NotFoundError extends Error { - public readonly body: { - statusCode: number; - name: string; - } = { - statusCode: 404, - name: 'Not found', - }; - - constructor(message: string | undefined) { - super(message); - } -} - describe('alert_details_route', () => { beforeEach(() => { jest.clearAllMocks(); @@ -58,11 +44,8 @@ describe('alert_details_route', () => { it('redirects to another page if fetched rule is an aliasMatch', async () => { await setup(); const rule = mockRule(); - const { loadAlert, resolveRule } = mockApis(); + const { resolveRule } = mockApis(); - loadAlert.mockImplementationOnce(async () => { - throw new NotFoundError('OMG'); - }); resolveRule.mockImplementationOnce(async () => ({ ...rule, id: 'new_id', @@ -70,17 +53,13 @@ describe('alert_details_route', () => { alias_target_id: rule.id, })); const wrapper = mountWithIntl( - + ); await act(async () => { await nextTick(); wrapper.update(); }); - expect(loadAlert).toHaveBeenCalledWith(rule.id); expect(resolveRule).toHaveBeenCalledWith(rule.id); expect((spacesMock as any).ui.redirectLegacyUrl).toHaveBeenCalledWith( `insightsAndAlerting/triggersActions/rule/new_id`, @@ -96,11 +75,8 @@ describe('alert_details_route', () => { name: 'type name', authorizedConsumers: ['consumer'], }; - const { loadAlert, loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); + const { loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); - loadAlert.mockImplementationOnce(async () => { - throw new NotFoundError('OMG'); - }); loadAlertTypes.mockImplementationOnce(async () => [ruleType]); loadActionTypes.mockImplementation(async () => []); resolveRule.mockImplementationOnce(async () => ({ @@ -112,7 +88,7 @@ describe('alert_details_route', () => { const wrapper = mountWithIntl( ); await act(async () => { @@ -120,7 +96,6 @@ describe('alert_details_route', () => { wrapper.update(); }); - expect(loadAlert).toHaveBeenCalledWith(rule.id); expect(resolveRule).toHaveBeenCalledWith(rule.id); expect((spacesMock as any).ui.components.getLegacyUrlConflict).toHaveBeenCalledWith({ currentObjectId: 'new_id', @@ -138,10 +113,10 @@ describe('getRuleData useEffect handler', () => { it('fetches rule', async () => { const rule = mockRule(); - const { loadAlert, loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); + const { loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); const { setAlert, setAlertType, setActionTypes } = mockStateSetter(); - loadAlert.mockImplementationOnce(async () => rule); + resolveRule.mockImplementationOnce(async () => rule); const toastNotifications = { addDanger: jest.fn(), @@ -149,7 +124,6 @@ describe('getRuleData useEffect handler', () => { await getRuleData( rule.id, - loadAlert, loadAlertTypes, resolveRule, loadActionTypes, @@ -159,8 +133,7 @@ describe('getRuleData useEffect handler', () => { toastNotifications ); - expect(loadAlert).toHaveBeenCalledWith(rule.id); - expect(resolveRule).not.toHaveBeenCalled(); + expect(resolveRule).toHaveBeenCalledWith(rule.id); expect(setAlert).toHaveBeenCalledWith(rule); }); @@ -184,10 +157,10 @@ describe('getRuleData useEffect handler', () => { id: rule.alertTypeId, name: 'type name', }; - const { loadAlert, loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); + const { loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); const { setAlert, setAlertType, setActionTypes } = mockStateSetter(); - loadAlert.mockImplementation(async () => rule); + resolveRule.mockImplementation(async () => rule); loadAlertTypes.mockImplementation(async () => [ruleType]); loadActionTypes.mockImplementation(async () => [connectorType]); @@ -197,7 +170,6 @@ describe('getRuleData useEffect handler', () => { await getRuleData( rule.id, - loadAlert, loadAlertTypes, resolveRule, loadActionTypes, @@ -209,58 +181,13 @@ describe('getRuleData useEffect handler', () => { expect(loadAlertTypes).toHaveBeenCalledTimes(1); expect(loadActionTypes).toHaveBeenCalledTimes(1); - expect(resolveRule).not.toHaveBeenCalled(); + expect(resolveRule).toHaveBeenCalled(); expect(setAlert).toHaveBeenCalledWith(rule); expect(setAlertType).toHaveBeenCalledWith(ruleType); expect(setActionTypes).toHaveBeenCalledWith([connectorType]); }); - it('fetches rule using resolve if initial GET results in a 404 error', async () => { - const connectorType = { - id: '.server-log', - name: 'Server log', - enabled: true, - }; - const rule = mockRule({ - actions: [ - { - group: '', - id: uuid.v4(), - actionTypeId: connectorType.id, - params: {}, - }, - ], - }); - - const { loadAlert, loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); - const { setAlert, setAlertType, setActionTypes } = mockStateSetter(); - - loadAlert.mockImplementationOnce(async () => { - throw new NotFoundError('OMG'); - }); - resolveRule.mockImplementationOnce(async () => rule); - - const toastNotifications = { - addDanger: jest.fn(), - } as unknown as ToastsApi; - await getRuleData( - rule.id, - loadAlert, - loadAlertTypes, - resolveRule, - loadActionTypes, - setAlert, - setAlertType, - setActionTypes, - toastNotifications - ); - - expect(loadAlert).toHaveBeenCalledWith(rule.id); - expect(resolveRule).toHaveBeenCalledWith(rule.id); - expect(setAlert).toHaveBeenCalledWith(rule); - }); - it('displays an error if fetching the rule results in a non-404 error', async () => { const connectorType = { id: '.server-log', @@ -278,10 +205,10 @@ describe('getRuleData useEffect handler', () => { ], }); - const { loadAlert, loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); + const { loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); const { setAlert, setAlertType, setActionTypes } = mockStateSetter(); - loadAlert.mockImplementation(async () => { + resolveRule.mockImplementation(async () => { throw new Error('OMG'); }); @@ -290,7 +217,6 @@ describe('getRuleData useEffect handler', () => { } as unknown as ToastsApi; await getRuleData( rule.id, - loadAlert, loadAlertTypes, resolveRule, loadActionTypes, @@ -322,10 +248,10 @@ describe('getRuleData useEffect handler', () => { ], }); - const { loadAlert, loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); + const { loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); const { setAlert, setAlertType, setActionTypes } = mockStateSetter(); - loadAlert.mockImplementation(async () => rule); + resolveRule.mockImplementation(async () => rule); loadAlertTypes.mockImplementation(async () => { throw new Error('OMG no rule type'); @@ -337,7 +263,6 @@ describe('getRuleData useEffect handler', () => { } as unknown as ToastsApi; await getRuleData( rule.id, - loadAlert, loadAlertTypes, resolveRule, loadActionTypes, @@ -373,10 +298,10 @@ describe('getRuleData useEffect handler', () => { name: 'type name', }; - const { loadAlert, loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); + const { loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); const { setAlert, setAlertType, setActionTypes } = mockStateSetter(); - loadAlert.mockImplementation(async () => rule); + resolveRule.mockImplementation(async () => rule); loadAlertTypes.mockImplementation(async () => [ruleType]); loadActionTypes.mockImplementation(async () => { @@ -388,7 +313,6 @@ describe('getRuleData useEffect handler', () => { } as unknown as ToastsApi; await getRuleData( rule.id, - loadAlert, loadAlertTypes, resolveRule, loadActionTypes, @@ -425,10 +349,10 @@ describe('getRuleData useEffect handler', () => { name: 'type name', }; - const { loadAlert, loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); + const { loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); const { setAlert, setAlertType, setActionTypes } = mockStateSetter(); - loadAlert.mockImplementation(async () => rule); + resolveRule.mockImplementation(async () => rule); loadAlertTypes.mockImplementation(async () => [ruleType]); loadActionTypes.mockImplementation(async () => [connectorType]); @@ -437,7 +361,6 @@ describe('getRuleData useEffect handler', () => { } as unknown as ToastsApi; await getRuleData( rule.id, - loadAlert, loadAlertTypes, resolveRule, loadActionTypes, @@ -485,10 +408,10 @@ describe('getRuleData useEffect handler', () => { name: 'type name', }; - const { loadAlert, loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); + const { loadAlertTypes, loadActionTypes, resolveRule } = mockApis(); const { setAlert, setAlertType, setActionTypes } = mockStateSetter(); - loadAlert.mockImplementation(async () => rule); + resolveRule.mockImplementation(async () => rule); loadAlertTypes.mockImplementation(async () => [ruleType]); loadActionTypes.mockImplementation(async () => [availableConnectorType]); @@ -497,7 +420,6 @@ describe('getRuleData useEffect handler', () => { } as unknown as ToastsApi; await getRuleData( rule.id, - loadAlert, loadAlertTypes, resolveRule, loadActionTypes, diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details_route.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details_route.tsx index 123d60bb9fea3..b530df986c277 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details_route.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_details/components/alert_details_route.tsx @@ -10,7 +10,7 @@ import React, { useState, useEffect } from 'react'; import { RouteComponentProps } from 'react-router-dom'; import { ToastsApi } from 'kibana/public'; import { EuiSpacer } from '@elastic/eui'; -import { Alert, AlertType, ActionType, ResolvedRule } from '../../../../types'; +import { AlertType, ActionType, ResolvedRule } from '../../../../types'; import { AlertDetailsWithApi as AlertDetails } from './alert_details'; import { throwIfAbsent, throwIfIsntContained } from '../../../lib/value_validators'; import { @@ -28,13 +28,12 @@ type AlertDetailsRouteProps = RouteComponentProps<{ ruleId: string; }> & Pick & - Pick; + Pick; export const AlertDetailsRoute: React.FunctionComponent = ({ match: { params: { ruleId }, }, - loadAlert, loadAlertTypes, loadActionTypes, resolveRule, @@ -47,14 +46,13 @@ export const AlertDetailsRoute: React.FunctionComponent const { basePath } = http; - const [alert, setAlert] = useState(null); + const [alert, setAlert] = useState(null); const [alertType, setAlertType] = useState(null); const [actionTypes, setActionTypes] = useState(null); const [refreshToken, requestRefresh] = React.useState(); useEffect(() => { getRuleData( ruleId, - loadAlert, loadAlertTypes, resolveRule, loadActionTypes, @@ -63,7 +61,7 @@ export const AlertDetailsRoute: React.FunctionComponent setActionTypes, toasts ); - }, [ruleId, http, loadActionTypes, loadAlert, loadAlertTypes, resolveRule, toasts, refreshToken]); + }, [ruleId, http, loadActionTypes, loadAlertTypes, resolveRule, toasts, refreshToken]); useEffect(() => { if (alert) { @@ -128,26 +126,16 @@ export const AlertDetailsRoute: React.FunctionComponent export async function getRuleData( ruleId: string, - loadAlert: AlertApis['loadAlert'], loadAlertTypes: AlertApis['loadAlertTypes'], resolveRule: AlertApis['resolveRule'], loadActionTypes: ActionApis['loadActionTypes'], - setAlert: React.Dispatch>, + setAlert: React.Dispatch>, setAlertType: React.Dispatch>, setActionTypes: React.Dispatch>, toasts: Pick ) { try { - let loadedRule: Alert | ResolvedRule; - try { - loadedRule = await loadAlert(ruleId); - } catch (err) { - // Try resolving this rule id if the error is a 404, otherwise re-throw - if (err?.body?.statusCode !== 404) { - throw err; - } - loadedRule = await resolveRule(ruleId); - } + const loadedRule: ResolvedRule = await resolveRule(ruleId); setAlert(loadedRule); const [loadedAlertType, loadedActionTypes] = await Promise.all([ From d822d6dc3229c45195fc2eea588e475b24c89dd7 Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Wed, 13 Oct 2021 13:45:53 -0700 Subject: [PATCH 24/35] Update kibana to EMS 7.16 (#114865) * Update kibana to EMS 7.16 * Update license override --- package.json | 2 +- src/dev/license_checker/config.ts | 2 +- src/plugins/maps_ems/common/index.ts | 2 +- yarn.lock | 20 ++++++++++---------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index e9aea2d2adda3..f526f357ff347 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "@elastic/charts": "37.0.0", "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath", "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.21", - "@elastic/ems-client": "7.15.0", + "@elastic/ems-client": "7.16.0", "@elastic/eui": "39.0.0", "@elastic/filesaver": "1.1.2", "@elastic/maki": "6.3.0", diff --git a/src/dev/license_checker/config.ts b/src/dev/license_checker/config.ts index 818ea3675194e..a4ae39848735e 100644 --- a/src/dev/license_checker/config.ts +++ b/src/dev/license_checker/config.ts @@ -74,7 +74,7 @@ export const DEV_ONLY_LICENSE_ALLOWED = ['MPL-2.0']; export const LICENSE_OVERRIDES = { 'jsts@1.6.2': ['Eclipse Distribution License - v 1.0'], // cf. https://github.com/bjornharrtell/jsts '@mapbox/jsonlint-lines-primitives@2.0.2': ['MIT'], // license in readme https://github.com/tmcw/jsonlint - '@elastic/ems-client@7.15.0': ['Elastic License 2.0'], + '@elastic/ems-client@7.16.0': ['Elastic License 2.0'], '@elastic/eui@39.0.0': ['SSPL-1.0 OR Elastic License 2.0'], 'language-subtag-registry@0.3.21': ['CC-BY-4.0'], // retired ODC‑By license https://github.com/mattcg/language-subtag-registry }; diff --git a/src/plugins/maps_ems/common/index.ts b/src/plugins/maps_ems/common/index.ts index f7d7ff1102e59..26fdb4fa795fe 100644 --- a/src/plugins/maps_ems/common/index.ts +++ b/src/plugins/maps_ems/common/index.ts @@ -10,7 +10,7 @@ export const TMS_IN_YML_ID = 'TMS in config/kibana.yml'; export const DEFAULT_EMS_FILE_API_URL = 'https://vector.maps.elastic.co'; export const DEFAULT_EMS_TILE_API_URL = 'https://tiles.maps.elastic.co'; -export const DEFAULT_EMS_LANDING_PAGE_URL = 'https://maps.elastic.co/v7.15'; +export const DEFAULT_EMS_LANDING_PAGE_URL = 'https://maps.elastic.co/v7.16'; export const DEFAULT_EMS_FONT_LIBRARY_URL = 'https://tiles.maps.elastic.co/fonts/{fontstack}/{range}.pbf'; diff --git a/yarn.lock b/yarn.lock index 05d03a637eaee..70e2a452e87dc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2389,10 +2389,10 @@ ms "^2.1.3" secure-json-parse "^2.4.0" -"@elastic/ems-client@7.15.0": - version "7.15.0" - resolved "https://registry.yarnpkg.com/@elastic/ems-client/-/ems-client-7.15.0.tgz#c101d7f83aa56463bcc385fd4eb883c6ea3ae9fc" - integrity sha512-BAAAVPhoaH6SGrfuO6U0MVRg4lvblhJ9VqYlMf3dZN9uDBB+12CUtb6t6Kavn5Tr3nS6X3tU/KKsuomo5RrEeQ== +"@elastic/ems-client@7.16.0": + version "7.16.0" + resolved "https://registry.yarnpkg.com/@elastic/ems-client/-/ems-client-7.16.0.tgz#92db94126bac0b95fbf156fe609f68979e7af4b6" + integrity sha512-NgMB5vqj6I7lxVsysrz6eB1EW6gsZj7SWWs79WSiiKQeNuRg82tJhvbHQnWezjIS4UKOtoGxZsg475EHVZB46g== dependencies: "@types/geojson" "^7946.0.7" "@types/lru-cache" "^5.1.0" @@ -2400,7 +2400,7 @@ "@types/topojson-specification" "^1.0.1" lodash "^4.17.15" lru-cache "^6.0.0" - semver "7.3.2" + semver "^7.3.2" topojson-client "^3.1.0" "@elastic/eslint-config-kibana@link:bazel-bin/packages/elastic-eslint-config-kibana": @@ -25810,16 +25810,16 @@ semver@7.0.0: resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== -semver@7.3.2, semver@^7.2.1, semver@^7.3.2, semver@~7.3.2: - version "7.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" - integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ== - semver@^6.0.0, semver@^6.1.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.2.1, semver@^7.3.2, semver@~7.3.2: + version "7.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" + integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ== + semver@^7.3.4, semver@^7.3.5: version "7.3.5" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" From 77ad8fe9912cd65fd6cd0ef1de23900fcccfe4e2 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Wed, 13 Oct 2021 14:21:50 -0700 Subject: [PATCH 25/35] docs: fix config names (#114903) --- docs/apm/api.asciidoc | 3 +-- docs/apm/troubleshooting.asciidoc | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/apm/api.asciidoc b/docs/apm/api.asciidoc index fe4c8a9280158..5f81a41e93df8 100644 --- a/docs/apm/api.asciidoc +++ b/docs/apm/api.asciidoc @@ -484,8 +484,7 @@ An example is below. [[api-create-apm-index-pattern]] ==== Customize the APM index pattern -As an alternative to updating <> in your `kibana.yml` configuration file, -you can use Kibana's <> to update the default APM index pattern on the fly. +Use Kibana's <> to update the default APM index pattern on the fly. The following example sets the default APM app index pattern to `some-other-pattern-*`: diff --git a/docs/apm/troubleshooting.asciidoc b/docs/apm/troubleshooting.asciidoc index 3736d21f44a5b..84cdb9876dc63 100644 --- a/docs/apm/troubleshooting.asciidoc +++ b/docs/apm/troubleshooting.asciidoc @@ -76,7 +76,7 @@ If you change the default, you must also configure the `setup.template.name` and See {apm-server-ref}/configuration-template.html[Load the Elasticsearch index template]. If the Elasticsearch index template has already been successfully loaded to the index, you can customize the indices that the APM app uses to display data. -Navigate to *APM* > *Settings* > *Indices*, and change all `xpack.apm.*Pattern` values to +Navigate to *APM* > *Settings* > *Indices*, and change all `xpack.apm.indices.*` values to include the new index pattern. For example: `customIndexName-*`. [float] From c737c393cf55518cc1bdda31b7a7da7e51403f36 Mon Sep 17 00:00:00 2001 From: Yuliia Naumenko Date: Wed, 13 Oct 2021 14:22:40 -0700 Subject: [PATCH 26/35] [Actions] Fixed actions telemetry for multiple namespaces usage (#114748) * [Actions] Fixed actions telemetry for multiple namespaces usage * fixed tests --- .../server/usage/actions_telemetry.test.ts | 159 ++++++++++-------- .../actions/server/usage/actions_telemetry.ts | 46 +++-- x-pack/plugins/actions/server/usage/task.ts | 21 +-- 3 files changed, 121 insertions(+), 105 deletions(-) diff --git a/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts b/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts index 4049d8fc3b594..0e6b7fff04451 100644 --- a/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts +++ b/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts @@ -116,7 +116,7 @@ Object { test('getInUseTotalCount', async () => { const mockEsClient = elasticsearchClientMock.createClusterClient().asScoped().asInternalUser; - mockEsClient.search.mockReturnValue( + mockEsClient.search.mockReturnValueOnce( // @ts-expect-error not full search response elasticsearchClientMock.createSuccessTransportRequestPromise({ aggregations: { @@ -134,28 +134,35 @@ Object { }, }) ); - const actionsBulkGet = jest.fn(); - actionsBulkGet.mockReturnValue({ - saved_objects: [ - { - id: '1', - attributes: { - actionTypeId: '.server-log', - }, - }, - { - id: '123', - attributes: { - actionTypeId: '.slack', - }, - }, - ], - }); - const telemetry = await getInUseTotalCount(mockEsClient, actionsBulkGet, 'test'); - expect(mockEsClient.search).toHaveBeenCalledTimes(1); - expect(actionsBulkGet).toHaveBeenCalledTimes(1); + mockEsClient.search.mockReturnValueOnce( + // @ts-expect-error not full search response + elasticsearchClientMock.createSuccessTransportRequestPromise({ + hits: { + hits: [ + { + _source: { + action: { + id: '1', + actionTypeId: '.server-log', + }, + }, + }, + { + _source: { + action: { + id: '2', + actionTypeId: '.slack', + }, + }, + }, + ], + }, + }) + ); + const telemetry = await getInUseTotalCount(mockEsClient, 'test'); + expect(mockEsClient.search).toHaveBeenCalledTimes(2); expect(telemetry).toMatchInlineSnapshot(` Object { "countByAlertHistoryConnectorType": 0, @@ -170,7 +177,7 @@ Object { test('getInUseTotalCount should count preconfigured alert history connector usage', async () => { const mockEsClient = elasticsearchClientMock.createClusterClient().asScoped().asInternalUser; - mockEsClient.search.mockReturnValue( + mockEsClient.search.mockReturnValueOnce( // @ts-expect-error not full search response elasticsearchClientMock.createSuccessTransportRequestPromise({ aggregations: { @@ -202,28 +209,34 @@ Object { }, }) ); - const actionsBulkGet = jest.fn(); - actionsBulkGet.mockReturnValue({ - saved_objects: [ - { - id: '1', - attributes: { - actionTypeId: '.server-log', - }, - }, - { - id: '123', - attributes: { - actionTypeId: '.slack', - }, + mockEsClient.search.mockReturnValueOnce( + // @ts-expect-error not full search response + elasticsearchClientMock.createSuccessTransportRequestPromise({ + hits: { + hits: [ + { + _source: { + action: { + id: '1', + actionTypeId: '.server-log', + }, + }, + }, + { + _source: { + action: { + id: '2', + actionTypeId: '.slack', + }, + }, + }, + ], }, - ], - }); - const telemetry = await getInUseTotalCount(mockEsClient, actionsBulkGet, 'test'); - - expect(mockEsClient.search).toHaveBeenCalledTimes(1); - expect(actionsBulkGet).toHaveBeenCalledTimes(1); + }) + ); + const telemetry = await getInUseTotalCount(mockEsClient, 'test'); + expect(mockEsClient.search).toHaveBeenCalledTimes(2); expect(telemetry).toMatchInlineSnapshot(` Object { "countByAlertHistoryConnectorType": 1, @@ -359,7 +372,7 @@ Object { test('getInUseTotalCount() accounts for preconfigured connectors', async () => { const mockEsClient = elasticsearchClientMock.createClusterClient().asScoped().asInternalUser; - mockEsClient.search.mockReturnValue( + mockEsClient.search.mockReturnValueOnce( // @ts-expect-error not full search response elasticsearchClientMock.createSuccessTransportRequestPromise({ aggregations: { @@ -399,34 +412,42 @@ Object { }, }) ); - const actionsBulkGet = jest.fn(); - actionsBulkGet.mockReturnValue({ - saved_objects: [ - { - id: '1', - attributes: { - actionTypeId: '.server-log', - }, - }, - { - id: '123', - attributes: { - actionTypeId: '.slack', - }, - }, - { - id: '456', - attributes: { - actionTypeId: '.email', - }, + mockEsClient.search.mockReturnValueOnce( + // @ts-expect-error not full search response + elasticsearchClientMock.createSuccessTransportRequestPromise({ + hits: { + hits: [ + { + _source: { + action: { + id: '1', + actionTypeId: '.server-log', + }, + }, + }, + { + _source: { + action: { + id: '2', + actionTypeId: '.slack', + }, + }, + }, + { + _source: { + action: { + id: '3', + actionTypeId: '.email', + }, + }, + }, + ], }, - ], - }); - const telemetry = await getInUseTotalCount(mockEsClient, actionsBulkGet, 'test'); - - expect(mockEsClient.search).toHaveBeenCalledTimes(1); - expect(actionsBulkGet).toHaveBeenCalledTimes(1); + }) + ); + const telemetry = await getInUseTotalCount(mockEsClient, 'test'); + expect(mockEsClient.search).toHaveBeenCalledTimes(2); expect(telemetry).toMatchInlineSnapshot(` Object { "countByAlertHistoryConnectorType": 1, diff --git a/x-pack/plugins/actions/server/usage/actions_telemetry.ts b/x-pack/plugins/actions/server/usage/actions_telemetry.ts index 544d6a411ccdc..4a3d0c70e535a 100644 --- a/x-pack/plugins/actions/server/usage/actions_telemetry.ts +++ b/x-pack/plugins/actions/server/usage/actions_telemetry.ts @@ -5,12 +5,7 @@ * 2.0. */ -import { - ElasticsearchClient, - SavedObjectsBaseOptions, - SavedObjectsBulkGetObject, - SavedObjectsBulkResponse, -} from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { AlertHistoryEsIndexConnectorId } from '../../common'; import { ActionResult, PreConfiguredAction } from '../types'; @@ -86,10 +81,6 @@ export async function getTotalCount( export async function getInUseTotalCount( esClient: ElasticsearchClient, - actionsBulkGet: ( - objects?: SavedObjectsBulkGetObject[] | undefined, - options?: SavedObjectsBaseOptions | undefined - ) => Promise>>>, kibanaIndex: string ): Promise<{ countTotal: number; @@ -259,15 +250,34 @@ export async function getInUseTotalCount( const preconfiguredActionsAggs = // @ts-expect-error aggegation type is not specified actionResults.aggregations.preconfigured_actions?.preconfiguredActionRefIds.value; - const bulkFilter = Object.entries(aggs.connectorIds).map(([key]) => ({ - id: key, - type: 'action', - fields: ['id', 'actionTypeId'], - })); - const actions = await actionsBulkGet(bulkFilter); - const countByActionTypeId = actions.saved_objects.reduce( + const { + body: { hits: actions }, + } = await esClient.search<{ + action: ActionResult; + }>({ + index: kibanaIndex, + _source_includes: ['action'], + body: { + query: { + bool: { + must: [ + { + term: { type: 'action' }, + }, + { + terms: { + _id: Object.entries(aggs.connectorIds).map(([key]) => `action:${key}`), + }, + }, + ], + }, + }, + }, + }); + const countByActionTypeId = actions.hits.reduce( (actionTypeCount: Record, action) => { - const alertTypeId = replaceFirstAndLastDotSymbols(action.attributes.actionTypeId); + const actionSource = action._source!; + const alertTypeId = replaceFirstAndLastDotSymbols(actionSource.action.actionTypeId); const currentCount = actionTypeCount[alertTypeId] !== undefined ? actionTypeCount[alertTypeId] : 0; actionTypeCount[alertTypeId] = currentCount + 1; diff --git a/x-pack/plugins/actions/server/usage/task.ts b/x-pack/plugins/actions/server/usage/task.ts index f37f830697eb5..7cbfb87dedda6 100644 --- a/x-pack/plugins/actions/server/usage/task.ts +++ b/x-pack/plugins/actions/server/usage/task.ts @@ -5,19 +5,14 @@ * 2.0. */ -import { - Logger, - CoreSetup, - SavedObjectsBulkGetObject, - SavedObjectsBaseOptions, -} from 'kibana/server'; +import { Logger, CoreSetup } from 'kibana/server'; import moment from 'moment'; import { RunContext, TaskManagerSetupContract, TaskManagerStartContract, } from '../../../task_manager/server'; -import { ActionResult, PreConfiguredAction } from '../types'; +import { PreConfiguredAction } from '../types'; import { getTotalCount, getInUseTotalCount } from './actions_telemetry'; export const TELEMETRY_TASK_TYPE = 'actions_telemetry'; @@ -83,22 +78,12 @@ export function telemetryTaskRunner( }, ]) => client.asInternalUser ); - const actionsBulkGet = ( - objects?: SavedObjectsBulkGetObject[], - options?: SavedObjectsBaseOptions - ) => { - return core - .getStartServices() - .then(([{ savedObjects }]) => - savedObjects.createInternalRepository(['action']).bulkGet(objects, options) - ); - }; return { async run() { const esClient = await getEsClient(); return Promise.all([ getTotalCount(esClient, kibanaIndex, preconfiguredActions), - getInUseTotalCount(esClient, actionsBulkGet, kibanaIndex), + getInUseTotalCount(esClient, kibanaIndex), ]) .then(([totalAggegations, totalInUse]) => { return { From 493b408673f708f7c58eb676194ac0cc90024dda Mon Sep 17 00:00:00 2001 From: Scotty Bollinger Date: Wed, 13 Oct 2021 17:14:14 -0500 Subject: [PATCH 27/35] [Workplace Search] Fix button order and remove extra source name label (#114899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove extra source title from Personal dashboard * Change button order to match other views We typically have the right-most button the Save button and the reset button to the left * Fix typo * Fix failing test EUI requires the name but we don’t want to dispaly it, so sending an empty string * Remove Synchronization nav items from Custom Source * Hide syncTriggerCallout for custom sources --- .../personal_dashboard_sidebar/private_sources_sidebar.tsx | 4 ++-- .../views/content_sources/components/overview.tsx | 5 +++-- .../views/content_sources/components/source_sub_nav.tsx | 2 +- .../workplace_search/views/security/security.tsx | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/personal_dashboard_sidebar/private_sources_sidebar.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/personal_dashboard_sidebar/private_sources_sidebar.tsx index 6cd7a10fc7ade..c8eaffbfbec10 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/personal_dashboard_sidebar/private_sources_sidebar.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/personal_dashboard_sidebar/private_sources_sidebar.tsx @@ -35,10 +35,10 @@ export const PrivateSourcesSidebar = () => { : PRIVATE_VIEW_ONLY_PAGE_DESCRIPTION; const { - contentSource: { id = '', name = '' }, + contentSource: { id = '' }, } = useValues(SourceLogic); - const navItems = [{ id, name, items: useSourceSubNav() }]; + const navItems = [{ id, name: '', items: useSourceSubNav() }]; return ( <> diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx index 899d9dceebe3e..9441f43dc253f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx @@ -119,6 +119,7 @@ export const Overview: React.FC = () => { const [isModalVisible, setIsModalVisible] = useState(false); const closeModal = () => setIsModalVisible(false); const handleSyncClick = () => setIsModalVisible(true); + const showSyncTriggerCallout = !custom && isIndexedSource && isOrganization; const onSyncConfirm = () => { initializeSourceSynchronization(id); @@ -491,7 +492,7 @@ export const Overview: React.FC = () => { @@ -586,7 +587,7 @@ export const Overview: React.FC = () => { )} )} - {isIndexedSource && isOrganization && syncTriggerCallout} + {showSyncTriggerCallout && syncTriggerCallout} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/source_sub_nav.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/source_sub_nav.tsx index cae1e8834cdd2..99597023303ff 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/source_sub_nav.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/source_sub_nav.tsx @@ -37,7 +37,7 @@ export const useSourceSubNav = () => { if (!id) return undefined; const isCustom = serviceType === CUSTOM_SERVICE_TYPE; - const showSynchronization = isIndexedSource && isOrganization; + const showSynchronization = isIndexedSource && isOrganization && !isCustom; const navItems: Array> = [ { diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/security/security.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/security/security.tsx index a971df8f89914..997d79f67cb13 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/security/security.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/security/security.tsx @@ -75,9 +75,6 @@ export const Security: React.FC = () => { }; const headerActions = [ - - {RESET_BUTTON} - , { > {SAVE_SETTINGS_BUTTON} , + + {RESET_BUTTON} + , ]; const allSourcesToggle = ( From e5576d688d843e895201d6993bdebe6d131ba15d Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Wed, 13 Oct 2021 18:46:04 -0400 Subject: [PATCH 28/35] [APM] Fixes incorrect index config names (#114901) (#114904) --- .../resources/base/bin/kibana-docker | 10 +++++----- .../integration/power_user/no_data_screen.ts | 20 +++++++++---------- .../plugins/apm/public/utils/testHelpers.tsx | 20 +++++++++---------- .../encrypted_saved_objects/mappings.json | 10 +++++----- .../mappings.json | 10 +++++----- .../es_archiver/key_rotation/mappings.json | 10 +++++----- .../action_task_params/mappings.json | 10 +++++----- .../es_archives/actions/mappings.json | 10 +++++----- .../es_archives/alerts_legacy/mappings.json | 10 +++++----- .../es_archives/canvas/reports/mappings.json | 10 +++++----- .../cases/migrations/7.10.0/mappings.json | 10 +++++----- .../cases/migrations/7.11.1/mappings.json | 10 +++++----- .../cases/migrations/7.13.2/mappings.json | 10 +++++----- .../7.13_user_actions/mappings.json | 10 +++++----- .../migrations/7.16.0_space/mappings.json | 10 +++++----- .../data/search_sessions/mappings.json | 10 +++++----- .../telemetry/agent_only/mappings.json | 10 +++++----- .../mappings.json | 10 +++++----- .../cloned_endpoint_installed/mappings.json | 10 +++++----- .../cloned_endpoint_uninstalled/mappings.json | 10 +++++----- .../endpoint_malware_disabled/mappings.json | 10 +++++----- .../endpoint_malware_enabled/mappings.json | 10 +++++----- .../endpoint_uninstalled/mappings.json | 10 +++++----- .../es_archives/fleet/agents/mappings.json | 10 +++++----- .../mappings.json | 10 +++++----- .../es_archives/lists/mappings.json | 10 +++++----- .../canvas_disallowed_url/mappings.json | 10 +++++----- .../ecommerce_kibana_spaces/mappings.json | 10 +++++----- .../reporting/hugedata/mappings.json | 10 +++++----- .../multi_index_kibana/mappings.json | 10 +++++----- .../migrations/mappings.json | 10 +++++----- .../timelines/7.15.0/mappings.json | 10 +++++----- .../timelines/7.15.0_space/mappings.json | 10 +++++----- .../visualize/default/mappings.json | 10 +++++----- 34 files changed, 180 insertions(+), 180 deletions(-) diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker index 9e7766ce16c9b..4a8f9df4c4044 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker @@ -211,12 +211,12 @@ kibana_vars=( xpack.alerts.invalidateApiKeysTask.interval xpack.alerts.invalidateApiKeysTask.removalDelay xpack.apm.enabled - xpack.apm.indices.errors - xpack.apm.indices.metrics + xpack.apm.indices.error + xpack.apm.indices.metric xpack.apm.indices.onboarding - xpack.apm.indices.sourcemaps - xpack.apm.indices.spans - xpack.apm.indices.transactions + xpack.apm.indices.sourcemap + xpack.apm.indices.span + xpack.apm.indices.transaction xpack.apm.maxServiceEnvironments xpack.apm.searchAggregatedTransactions xpack.apm.serviceMapEnabled diff --git a/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/no_data_screen.ts b/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/no_data_screen.ts index 47eba11e6f6fb..56704d63a42f1 100644 --- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/no_data_screen.ts +++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/no_data_screen.ts @@ -19,12 +19,12 @@ describe('No data screen', () => { url: apmIndicesSaveURL, method: 'POST', body: { - sourcemaps: 'foo-*', - errors: 'foo-*', + sourcemap: 'foo-*', + error: 'foo-*', onboarding: 'foo-*', - spans: 'foo-*', - transactions: 'foo-*', - metrics: 'foo-*', + span: 'foo-*', + transaction: 'foo-*', + metric: 'foo-*', }, headers: { 'kbn-xsrf': true, @@ -49,12 +49,12 @@ describe('No data screen', () => { url: apmIndicesSaveURL, method: 'POST', body: { - sourcemaps: '', - errors: '', + sourcemap: '', + error: '', onboarding: '', - spans: '', - transactions: '', - metrics: '', + span: '', + transaction: '', + metric: '', }, headers: { 'kbn-xsrf': true }, auth: { user: 'apm_power_user', pass: 'changeme' }, diff --git a/x-pack/plugins/apm/public/utils/testHelpers.tsx b/x-pack/plugins/apm/public/utils/testHelpers.tsx index 2203bc63f68cd..9ce7d2e4a52d9 100644 --- a/x-pack/plugins/apm/public/utils/testHelpers.tsx +++ b/x-pack/plugins/apm/public/utils/testHelpers.tsx @@ -119,12 +119,12 @@ interface MockSetup { config: APMConfig; uiFilters: UxUIFilters; indices: { - sourcemaps: string; - errors: string; + sourcemap: string; + error: string; onboarding: string; - spans: string; - transactions: string; - metrics: string; + span: string; + transaction: string; + metric: string; apmAgentConfigurationIndex: string; apmCustomLinkIndex: string; }; @@ -176,12 +176,12 @@ export async function inspectSearchParams( ) as APMConfig, uiFilters: {}, indices: { - sourcemaps: 'myIndex', - errors: 'myIndex', + sourcemap: 'myIndex', + error: 'myIndex', onboarding: 'myIndex', - spans: 'myIndex', - transactions: 'myIndex', - metrics: 'myIndex', + span: 'myIndex', + transaction: 'myIndex', + metric: 'myIndex', apmAgentConfigurationIndex: 'myIndex', apmCustomLinkIndex: 'myIndex', }, diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects/mappings.json b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects/mappings.json index daa99c4d71967..be99452707814 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects/mappings.json +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects/mappings.json @@ -189,22 +189,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json index 4e41d8cb72fb5..dfcf3155b67ca 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json @@ -216,22 +216,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/key_rotation/mappings.json b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/key_rotation/mappings.json index 9164430730216..72f66db35cec6 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/key_rotation/mappings.json +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/key_rotation/mappings.json @@ -214,22 +214,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/action_task_params/mappings.json b/x-pack/test/functional/es_archives/action_task_params/mappings.json index 874886647e6d6..d28c1504d3eed 100644 --- a/x-pack/test/functional/es_archives/action_task_params/mappings.json +++ b/x-pack/test/functional/es_archives/action_task_params/mappings.json @@ -206,22 +206,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/actions/mappings.json b/x-pack/test/functional/es_archives/actions/mappings.json index 786005d1ab6a6..a0da38c85f724 100644 --- a/x-pack/test/functional/es_archives/actions/mappings.json +++ b/x-pack/test/functional/es_archives/actions/mappings.json @@ -202,22 +202,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/alerts_legacy/mappings.json b/x-pack/test/functional/es_archives/alerts_legacy/mappings.json index 9c856a829a343..6e40f811e1af4 100644 --- a/x-pack/test/functional/es_archives/alerts_legacy/mappings.json +++ b/x-pack/test/functional/es_archives/alerts_legacy/mappings.json @@ -198,22 +198,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/canvas/reports/mappings.json b/x-pack/test/functional/es_archives/canvas/reports/mappings.json index 51c48857d2481..d6c3f1b26a430 100644 --- a/x-pack/test/functional/es_archives/canvas/reports/mappings.json +++ b/x-pack/test/functional/es_archives/canvas/reports/mappings.json @@ -229,22 +229,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/cases/migrations/7.10.0/mappings.json b/x-pack/test/functional/es_archives/cases/migrations/7.10.0/mappings.json index bd0b2c4e9ad27..b1b6c468c3945 100644 --- a/x-pack/test/functional/es_archives/cases/migrations/7.10.0/mappings.json +++ b/x-pack/test/functional/es_archives/cases/migrations/7.10.0/mappings.json @@ -199,22 +199,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/cases/migrations/7.11.1/mappings.json b/x-pack/test/functional/es_archives/cases/migrations/7.11.1/mappings.json index 2da75330be93a..94648d407a89b 100644 --- a/x-pack/test/functional/es_archives/cases/migrations/7.11.1/mappings.json +++ b/x-pack/test/functional/es_archives/cases/migrations/7.11.1/mappings.json @@ -254,22 +254,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/cases/migrations/7.13.2/mappings.json b/x-pack/test/functional/es_archives/cases/migrations/7.13.2/mappings.json index 62fec329aa40f..6f4a2df3a7543 100644 --- a/x-pack/test/functional/es_archives/cases/migrations/7.13.2/mappings.json +++ b/x-pack/test/functional/es_archives/cases/migrations/7.13.2/mappings.json @@ -260,22 +260,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/cases/migrations/7.13_user_actions/mappings.json b/x-pack/test/functional/es_archives/cases/migrations/7.13_user_actions/mappings.json index 8a9c1a626e652..7026e50cdb658 100644 --- a/x-pack/test/functional/es_archives/cases/migrations/7.13_user_actions/mappings.json +++ b/x-pack/test/functional/es_archives/cases/migrations/7.13_user_actions/mappings.json @@ -261,22 +261,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/cases/migrations/7.16.0_space/mappings.json b/x-pack/test/functional/es_archives/cases/migrations/7.16.0_space/mappings.json index 4b8f2c7103b20..d7422d4236598 100644 --- a/x-pack/test/functional/es_archives/cases/migrations/7.16.0_space/mappings.json +++ b/x-pack/test/functional/es_archives/cases/migrations/7.16.0_space/mappings.json @@ -275,22 +275,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/data/search_sessions/mappings.json b/x-pack/test/functional/es_archives/data/search_sessions/mappings.json index 1203e1d892d56..07c2c88b9f38f 100644 --- a/x-pack/test/functional/es_archives/data/search_sessions/mappings.json +++ b/x-pack/test/functional/es_archives/data/search_sessions/mappings.json @@ -169,22 +169,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/endpoint/telemetry/agent_only/mappings.json b/x-pack/test/functional/es_archives/endpoint/telemetry/agent_only/mappings.json index daab89b69483e..dd65c3977e1b7 100644 --- a/x-pack/test/functional/es_archives/endpoint/telemetry/agent_only/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/telemetry/agent_only/mappings.json @@ -198,22 +198,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_different_states/mappings.json b/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_different_states/mappings.json index c133c3fec76e2..eecc8ee5d8870 100644 --- a/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_different_states/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_different_states/mappings.json @@ -199,22 +199,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_installed/mappings.json b/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_installed/mappings.json index c133c3fec76e2..eecc8ee5d8870 100644 --- a/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_installed/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_installed/mappings.json @@ -199,22 +199,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_uninstalled/mappings.json b/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_uninstalled/mappings.json index c133c3fec76e2..eecc8ee5d8870 100644 --- a/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_uninstalled/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_uninstalled/mappings.json @@ -199,22 +199,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_disabled/mappings.json b/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_disabled/mappings.json index daab89b69483e..dd65c3977e1b7 100644 --- a/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_disabled/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_disabled/mappings.json @@ -198,22 +198,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_enabled/mappings.json b/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_enabled/mappings.json index daab89b69483e..dd65c3977e1b7 100644 --- a/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_enabled/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_enabled/mappings.json @@ -198,22 +198,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_uninstalled/mappings.json b/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_uninstalled/mappings.json index daab89b69483e..dd65c3977e1b7 100644 --- a/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_uninstalled/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_uninstalled/mappings.json @@ -198,22 +198,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/fleet/agents/mappings.json b/x-pack/test/functional/es_archives/fleet/agents/mappings.json index b2f5392ebd23a..24b4a66624305 100644 --- a/x-pack/test/functional/es_archives/fleet/agents/mappings.json +++ b/x-pack/test/functional/es_archives/fleet/agents/mappings.json @@ -189,22 +189,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/kibana_scripted_fields_on_logstash/mappings.json b/x-pack/test/functional/es_archives/kibana_scripted_fields_on_logstash/mappings.json index 7853368bc37d5..e0dd6d90eacb4 100644 --- a/x-pack/test/functional/es_archives/kibana_scripted_fields_on_logstash/mappings.json +++ b/x-pack/test/functional/es_archives/kibana_scripted_fields_on_logstash/mappings.json @@ -181,22 +181,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/lists/mappings.json b/x-pack/test/functional/es_archives/lists/mappings.json index e23c5ad224506..e687285f91b29 100644 --- a/x-pack/test/functional/es_archives/lists/mappings.json +++ b/x-pack/test/functional/es_archives/lists/mappings.json @@ -196,22 +196,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/reporting/canvas_disallowed_url/mappings.json b/x-pack/test/functional/es_archives/reporting/canvas_disallowed_url/mappings.json index a35e4c8e07e97..e67abaf2032c7 100644 --- a/x-pack/test/functional/es_archives/reporting/canvas_disallowed_url/mappings.json +++ b/x-pack/test/functional/es_archives/reporting/canvas_disallowed_url/mappings.json @@ -193,22 +193,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/reporting/ecommerce_kibana_spaces/mappings.json b/x-pack/test/functional/es_archives/reporting/ecommerce_kibana_spaces/mappings.json index 9d11349819d68..0fe9a18ce2201 100644 --- a/x-pack/test/functional/es_archives/reporting/ecommerce_kibana_spaces/mappings.json +++ b/x-pack/test/functional/es_archives/reporting/ecommerce_kibana_spaces/mappings.json @@ -214,22 +214,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/reporting/hugedata/mappings.json b/x-pack/test/functional/es_archives/reporting/hugedata/mappings.json index 02a212d65cc1a..d1cb75c1f5150 100644 --- a/x-pack/test/functional/es_archives/reporting/hugedata/mappings.json +++ b/x-pack/test/functional/es_archives/reporting/hugedata/mappings.json @@ -170,22 +170,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/reporting/multi_index_kibana/mappings.json b/x-pack/test/functional/es_archives/reporting/multi_index_kibana/mappings.json index f6b5df41938fb..69c6cbc3b46b5 100644 --- a/x-pack/test/functional/es_archives/reporting/multi_index_kibana/mappings.json +++ b/x-pack/test/functional/es_archives/reporting/multi_index_kibana/mappings.json @@ -172,22 +172,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/security_solution/migrations/mappings.json b/x-pack/test/functional/es_archives/security_solution/migrations/mappings.json index fa49916066a1a..8728ec4ad74a1 100644 --- a/x-pack/test/functional/es_archives/security_solution/migrations/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/migrations/mappings.json @@ -272,22 +272,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/security_solution/timelines/7.15.0/mappings.json b/x-pack/test/functional/es_archives/security_solution/timelines/7.15.0/mappings.json index 2e6a9bcee3d8c..7292878908cab 100644 --- a/x-pack/test/functional/es_archives/security_solution/timelines/7.15.0/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timelines/7.15.0/mappings.json @@ -276,22 +276,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/security_solution/timelines/7.15.0_space/mappings.json b/x-pack/test/functional/es_archives/security_solution/timelines/7.15.0_space/mappings.json index 682b241c126f4..45206c84b69de 100644 --- a/x-pack/test/functional/es_archives/security_solution/timelines/7.15.0_space/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timelines/7.15.0_space/mappings.json @@ -273,22 +273,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } diff --git a/x-pack/test/functional/es_archives/visualize/default/mappings.json b/x-pack/test/functional/es_archives/visualize/default/mappings.json index 2d761064d0a46..abf6bcfa04c80 100644 --- a/x-pack/test/functional/es_archives/visualize/default/mappings.json +++ b/x-pack/test/functional/es_archives/visualize/default/mappings.json @@ -254,22 +254,22 @@ }, "apm-indices": { "properties": { - "errors": { + "error": { "type": "keyword" }, - "metrics": { + "metric": { "type": "keyword" }, "onboarding": { "type": "keyword" }, - "sourcemaps": { + "sourcemap": { "type": "keyword" }, - "spans": { + "span": { "type": "keyword" }, - "transactions": { + "transaction": { "type": "keyword" } } From f8cbbbb99fbed1a6dfcacbe13aee7e922030fb36 Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Wed, 13 Oct 2021 20:11:53 -0400 Subject: [PATCH 29/35] [Controls] Redux Toolkit and Embeddable Redux Wrapper (#114371) Use new redux wrapper for control group management and upgrade all control group methods to use redux wrapper. Get order of controls from embeddable input, set up preconfigured story. Co-authored-by: andreadelrio --- package.json | 3 + .../__stories__/controls_service_stub.ts | 2 +- .../__stories__/input_controls.stories.tsx | 68 +++++- .../storybook_control_factories.ts | 27 +++ .../control_frame/control_frame_strings.ts | 22 -- .../component}/control_frame_component.tsx | 47 ++-- .../component/control_group_component.tsx | 121 +++++----- .../component/control_group_sortable_item.tsx | 114 ++++----- .../control_group/control_group_container.tsx | 224 ------------------ .../control_group/control_group_strings.ts | 63 +++-- ...{manage_control.tsx => control_editor.tsx} | 2 +- .../control_group/editor/create_control.tsx | 158 ++++++++++++ .../control_group/editor/edit_control.tsx | 127 ++++++++++ .../editor/edit_control_group.tsx | 124 ++++++++++ .../editor/forward_all_context.tsx | 36 +++ .../editor/manage_control_group_component.tsx | 113 --------- .../embeddable/control_group_container.tsx | 77 ++++++ .../control_group_container_factory.ts | 19 +- .../state/control_group_reducers.ts | 48 ++++ .../controls/control_group/types.ts | 4 +- .../options_list/options_list_embeddable.tsx | 2 +- .../components/controls/controls_service.ts | 6 +- .../controls/hooks/use_child_embeddable.ts | 11 +- .../public/components/controls/types.ts | 38 +-- .../generic_embeddable_store.ts | 40 ++++ .../redux_embeddable_context.ts | 73 ++++++ .../redux_embeddable_wrapper.tsx | 162 +++++++++++++ .../components/redux_embeddables/types.ts | 62 +++++ src/plugins/presentation_util/public/mocks.ts | 1 + .../presentation_util/public/plugin.ts | 1 + .../public/services/controls.ts | 85 +++++++ .../public/services/index.ts | 3 + .../index.ts => services/kibana/controls.ts} | 6 + .../public/services/kibana/index.ts | 2 + .../public/services/storybook/controls.ts | 13 + .../public/services/storybook/index.ts | 11 +- .../public/services/stub/controls.ts | 13 + .../public/services/stub/index.ts | 3 +- src/plugins/presentation_util/public/types.ts | 2 + 39 files changed, 1308 insertions(+), 625 deletions(-) create mode 100644 src/plugins/presentation_util/public/components/controls/__stories__/storybook_control_factories.ts delete mode 100644 src/plugins/presentation_util/public/components/controls/control_frame/control_frame_strings.ts rename src/plugins/presentation_util/public/components/controls/{control_frame => control_group/component}/control_frame_component.tsx (71%) delete mode 100644 src/plugins/presentation_util/public/components/controls/control_group/control_group_container.tsx rename src/plugins/presentation_util/public/components/controls/control_group/editor/{manage_control.tsx => control_editor.tsx} (99%) create mode 100644 src/plugins/presentation_util/public/components/controls/control_group/editor/create_control.tsx create mode 100644 src/plugins/presentation_util/public/components/controls/control_group/editor/edit_control.tsx create mode 100644 src/plugins/presentation_util/public/components/controls/control_group/editor/edit_control_group.tsx create mode 100644 src/plugins/presentation_util/public/components/controls/control_group/editor/forward_all_context.tsx delete mode 100644 src/plugins/presentation_util/public/components/controls/control_group/editor/manage_control_group_component.tsx create mode 100644 src/plugins/presentation_util/public/components/controls/control_group/embeddable/control_group_container.tsx rename src/plugins/presentation_util/public/components/controls/control_group/{ => embeddable}/control_group_container_factory.ts (71%) create mode 100644 src/plugins/presentation_util/public/components/controls/control_group/state/control_group_reducers.ts create mode 100644 src/plugins/presentation_util/public/components/redux_embeddables/generic_embeddable_store.ts create mode 100644 src/plugins/presentation_util/public/components/redux_embeddables/redux_embeddable_context.ts create mode 100644 src/plugins/presentation_util/public/components/redux_embeddables/redux_embeddable_wrapper.tsx create mode 100644 src/plugins/presentation_util/public/components/redux_embeddables/types.ts create mode 100644 src/plugins/presentation_util/public/services/controls.ts rename src/plugins/presentation_util/public/{components/controls/index.ts => services/kibana/controls.ts} (54%) create mode 100644 src/plugins/presentation_util/public/services/storybook/controls.ts create mode 100644 src/plugins/presentation_util/public/services/stub/controls.ts diff --git a/package.json b/package.json index f526f357ff347..6e4a37863bc82 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,9 @@ "yarn": "^1.21.1" }, "dependencies": { + "@dnd-kit/core": "^3.1.1", + "@dnd-kit/sortable": "^4.0.0", + "@dnd-kit/utilities": "^2.0.0", "@babel/runtime": "^7.15.4", "@dnd-kit/core": "^3.1.1", "@dnd-kit/sortable": "^4.0.0", diff --git a/src/plugins/presentation_util/public/components/controls/__stories__/controls_service_stub.ts b/src/plugins/presentation_util/public/components/controls/__stories__/controls_service_stub.ts index 59e7a44a83a17..faaa155249949 100644 --- a/src/plugins/presentation_util/public/components/controls/__stories__/controls_service_stub.ts +++ b/src/plugins/presentation_util/public/components/controls/__stories__/controls_service_stub.ts @@ -6,8 +6,8 @@ * Side Public License, v 1. */ -import { InputControlFactory } from '../types'; import { ControlsService } from '../controls_service'; +import { InputControlFactory } from '../../../services/controls'; import { flightFields, getEuiSelectableOptions } from './flights'; import { OptionsListEmbeddableFactory } from '../control_types/options_list'; diff --git a/src/plugins/presentation_util/public/components/controls/__stories__/input_controls.stories.tsx b/src/plugins/presentation_util/public/components/controls/__stories__/input_controls.stories.tsx index 2a463fece18da..66f1d8b36399e 100644 --- a/src/plugins/presentation_util/public/components/controls/__stories__/input_controls.stories.tsx +++ b/src/plugins/presentation_util/public/components/controls/__stories__/input_controls.stories.tsx @@ -10,9 +10,14 @@ import React, { useEffect, useMemo } from 'react'; import uuid from 'uuid'; import { decorators } from './decorators'; -import { providers } from '../../../services/storybook'; -import { getControlsServiceStub } from './controls_service_stub'; -import { ControlGroupContainerFactory } from '../control_group/control_group_container_factory'; +import { pluginServices, registry } from '../../../services/storybook'; +import { populateStorybookControlFactories } from './storybook_control_factories'; +import { ControlGroupContainerFactory } from '../control_group/embeddable/control_group_container_factory'; +import { ControlsPanels } from '../control_group/types'; +import { + OptionsListEmbeddableInput, + OPTIONS_LIST_CONTROL, +} from '../control_types/options_list/options_list_embeddable'; export default { title: 'Controls', @@ -20,17 +25,15 @@ export default { decorators, }; -const ControlGroupStoryComponent = () => { +const EmptyControlGroupStoryComponent = ({ panels }: { panels?: ControlsPanels }) => { const embeddableRoot: React.RefObject = useMemo(() => React.createRef(), []); - providers.overlays.start({}); - const overlays = providers.overlays.getService(); - - const controlsServiceStub = getControlsServiceStub(); + pluginServices.setRegistry(registry.start({})); + populateStorybookControlFactories(pluginServices.getServices().controls); useEffect(() => { (async () => { - const factory = new ControlGroupContainerFactory(controlsServiceStub, overlays); + const factory = new ControlGroupContainerFactory(); const controlGroupContainerEmbeddable = await factory.create({ inheritParentState: { useQuery: false, @@ -38,16 +41,57 @@ const ControlGroupStoryComponent = () => { useTimerange: false, }, controlStyle: 'oneLine', + panels: panels ?? {}, id: uuid.v4(), - panels: {}, }); if (controlGroupContainerEmbeddable && embeddableRoot.current) { controlGroupContainerEmbeddable.render(embeddableRoot.current); } })(); - }, [embeddableRoot, controlsServiceStub, overlays]); + }, [embeddableRoot, panels]); return
; }; -export const ControlGroupStory = () => ; +export const EmptyControlGroupStory = () => ; +export const ConfiguredControlGroupStory = () => ( + +); diff --git a/src/plugins/presentation_util/public/components/controls/__stories__/storybook_control_factories.ts b/src/plugins/presentation_util/public/components/controls/__stories__/storybook_control_factories.ts new file mode 100644 index 0000000000000..3048adc74d8c7 --- /dev/null +++ b/src/plugins/presentation_util/public/components/controls/__stories__/storybook_control_factories.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { flightFields, getEuiSelectableOptions } from './flights'; +import { OptionsListEmbeddableFactory } from '../control_types/options_list'; +import { InputControlFactory, PresentationControlsService } from '../../../services/controls'; + +export const populateStorybookControlFactories = ( + controlsServiceStub: PresentationControlsService +) => { + const optionsListFactoryStub = new OptionsListEmbeddableFactory( + ({ field, search }) => + new Promise((r) => setTimeout(() => r(getEuiSelectableOptions(field, search)), 500)), + () => Promise.resolve(['demo data flights']), + () => Promise.resolve(flightFields) + ); + + // cast to unknown because the stub cannot use the embeddable start contract to transform the EmbeddableFactoryDefinition into an EmbeddableFactory + const optionsListControlFactory = optionsListFactoryStub as unknown as InputControlFactory; + optionsListControlFactory.getDefaultInput = () => ({}); + controlsServiceStub.registerInputControlType(optionsListControlFactory); +}; diff --git a/src/plugins/presentation_util/public/components/controls/control_frame/control_frame_strings.ts b/src/plugins/presentation_util/public/components/controls/control_frame/control_frame_strings.ts deleted file mode 100644 index 5f9e89aa797cb..0000000000000 --- a/src/plugins/presentation_util/public/components/controls/control_frame/control_frame_strings.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { i18n } from '@kbn/i18n'; - -export const ControlFrameStrings = { - floatingActions: { - getEditButtonTitle: () => - i18n.translate('presentationUtil.inputControls.controlGroup.floatingActions.editTitle', { - defaultMessage: 'Manage control', - }), - getRemoveButtonTitle: () => - i18n.translate('presentationUtil.inputControls.controlGroup.floatingActions.removeTitle', { - defaultMessage: 'Remove control', - }), - }, -}; diff --git a/src/plugins/presentation_util/public/components/controls/control_frame/control_frame_component.tsx b/src/plugins/presentation_util/public/components/controls/control_group/component/control_frame_component.tsx similarity index 71% rename from src/plugins/presentation_util/public/components/controls/control_frame/control_frame_component.tsx rename to src/plugins/presentation_util/public/components/controls/control_group/component/control_frame_component.tsx index 240beea13b0e2..103ce6dd0e27c 100644 --- a/src/plugins/presentation_util/public/components/controls/control_frame/control_frame_component.tsx +++ b/src/plugins/presentation_util/public/components/controls/control_group/component/control_frame_component.tsx @@ -15,32 +15,28 @@ import { EuiFormRow, EuiToolTip, } from '@elastic/eui'; -import { ControlGroupContainer } from '../control_group/control_group_container'; -import { useChildEmbeddable } from '../hooks/use_child_embeddable'; -import { ControlStyle } from '../types'; -import { ControlFrameStrings } from './control_frame_strings'; + +import { ControlGroupInput } from '../types'; +import { EditControlButton } from '../editor/edit_control'; +import { useChildEmbeddable } from '../../hooks/use_child_embeddable'; +import { useReduxContainerContext } from '../../../redux_embeddables/redux_embeddable_context'; +import { ControlGroupStrings } from '../control_group_strings'; export interface ControlFrameProps { - container: ControlGroupContainer; customPrepend?: JSX.Element; - controlStyle: ControlStyle; enableActions?: boolean; - onRemove?: () => void; embeddableId: string; - onEdit?: () => void; } -export const ControlFrame = ({ - customPrepend, - enableActions, - embeddableId, - controlStyle, - container, - onRemove, - onEdit, -}: ControlFrameProps) => { +export const ControlFrame = ({ customPrepend, enableActions, embeddableId }: ControlFrameProps) => { const embeddableRoot: React.RefObject = useMemo(() => React.createRef(), []); - const embeddable = useChildEmbeddable({ container, embeddableId }); + const { + useEmbeddableSelector, + containerActions: { untilEmbeddableLoaded, removeEmbeddable }, + } = useReduxContainerContext(); + const { controlStyle } = useEmbeddableSelector((state) => state); + + const embeddable = useChildEmbeddable({ untilEmbeddableLoaded, embeddableId }); const [title, setTitle] = useState(); @@ -61,18 +57,13 @@ export const ControlFrame = ({ 'controlFrame--floatingActions-oneLine': !usingTwoLineLayout, })} > - - + + - + removeEmbeddable(embeddableId)} iconType="cross" color="danger" /> diff --git a/src/plugins/presentation_util/public/components/controls/control_group/component/control_group_component.tsx b/src/plugins/presentation_util/public/components/controls/control_group/component/control_group_component.tsx index d683c0749d98d..4d5e8bc270e23 100644 --- a/src/plugins/presentation_util/public/components/controls/control_group/component/control_group_component.tsx +++ b/src/plugins/presentation_util/public/components/controls/control_group/component/control_group_component.tsx @@ -9,7 +9,7 @@ import '../control_group.scss'; import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiToolTip } from '@elastic/eui'; -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useMemo, useState } from 'react'; import classNames from 'classnames'; import { arrayMove, @@ -29,46 +29,51 @@ import { LayoutMeasuringStrategy, } from '@dnd-kit/core'; +import { ControlGroupInput } from '../types'; +import { pluginServices } from '../../../../services'; import { ControlGroupStrings } from '../control_group_strings'; -import { ControlGroupContainer } from '../control_group_container'; +import { CreateControlButton } from '../editor/create_control'; +import { EditControlGroup } from '../editor/edit_control_group'; +import { forwardAllContext } from '../editor/forward_all_context'; import { ControlClone, SortableControl } from './control_group_sortable_item'; -import { OPTIONS_LIST_CONTROL } from '../../control_types/options_list/options_list_embeddable'; +import { useReduxContainerContext } from '../../../redux_embeddables/redux_embeddable_context'; +import { controlGroupReducers } from '../state/control_group_reducers'; -interface ControlGroupProps { - controlGroupContainer: ControlGroupContainer; -} +export const ControlGroup = () => { + // Presentation Services Context + const { overlays } = pluginServices.getHooks(); + const { openFlyout } = overlays.useService(); -export const ControlGroup = ({ controlGroupContainer }: ControlGroupProps) => { - const [controlIds, setControlIds] = useState([]); + // Redux embeddable container Context + const reduxContainerContext = useReduxContainerContext< + ControlGroupInput, + typeof controlGroupReducers + >(); + const { + useEmbeddableSelector, + useEmbeddableDispatch, + actions: { setControlOrders }, + } = reduxContainerContext; + const dispatch = useEmbeddableDispatch(); - // sync controlIds every time input panels change - useEffect(() => { - const subscription = controlGroupContainer.getInput$().subscribe(() => { - setControlIds((currentIds) => { - // sync control Ids with panels from container input. - const { panels } = controlGroupContainer.getInput(); - const newIds: string[] = []; - const allIds = [...currentIds, ...Object.keys(panels)]; - allIds.forEach((id) => { - const currentIndex = currentIds.indexOf(id); - if (!panels[id] && currentIndex !== -1) { - currentIds.splice(currentIndex, 1); - } - if (currentIndex === -1 && Boolean(panels[id])) { - newIds.push(id); - } - }); - return [...currentIds, ...newIds]; - }); - }); - return () => subscription.unsubscribe(); - }, [controlGroupContainer]); + // current state + const { panels } = useEmbeddableSelector((state) => state); - const [draggingId, setDraggingId] = useState(null); + const idsInOrder = useMemo( + () => + Object.values(panels) + .sort((a, b) => (a.order > b.order ? 1 : -1)) + .reduce((acc, panel) => { + acc.push(panel.explicitInput.id); + return acc; + }, [] as string[]), + [panels] + ); + const [draggingId, setDraggingId] = useState(null); const draggingIndex = useMemo( - () => (draggingId ? controlIds.indexOf(draggingId) : -1), - [controlIds, draggingId] + () => (draggingId ? idsInOrder.indexOf(draggingId) : -1), + [idsInOrder, draggingId] ); const sensors = useSensors( @@ -78,10 +83,10 @@ export const ControlGroup = ({ controlGroupContainer }: ControlGroupProps) => { const onDragEnd = ({ over }: DragEndEvent) => { if (over) { - const overIndex = controlIds.indexOf(over.id); + const overIndex = idsInOrder.indexOf(over.id); if (draggingIndex !== overIndex) { const newIndex = overIndex; - setControlIds((currentControlIds) => arrayMove(currentControlIds, draggingIndex, newIndex)); + dispatch(setControlOrders({ ids: arrayMove([...idsInOrder], draggingIndex, newIndex) })); } } setDraggingId(null); @@ -100,36 +105,26 @@ export const ControlGroup = ({ controlGroupContainer }: ControlGroupProps) => { strategy: LayoutMeasuringStrategy.Always, }} > - + - {controlIds.map((controlId, index) => ( - controlGroupContainer.editControl(controlId)} - onRemove={() => controlGroupContainer.removeEmbeddable(controlId)} - dragInfo={{ index, draggingIndex }} - container={controlGroupContainer} - controlStyle={controlGroupContainer.getInput().controlStyle} - embeddableId={controlId} - width={controlGroupContainer.getInput().panels[controlId].width} - key={controlId} - /> - ))} + {idsInOrder.map( + (controlId, index) => + panels[controlId] && ( + + ) + )} - - {draggingId ? ( - - ) : null} - + {draggingId ? : null} @@ -141,19 +136,15 @@ export const ControlGroup = ({ controlGroupContainer }: ControlGroupProps) => { iconType="gear" color="text" data-test-subj="inputControlsSortingButton" - onClick={controlGroupContainer.editControlGroup} + onClick={() => + openFlyout(forwardAllContext(, reduxContainerContext)) + } /> - controlGroupContainer.createNewControl(OPTIONS_LIST_CONTROL)} // use popover when there are multiple types of control - /> + diff --git a/src/plugins/presentation_util/public/components/controls/control_group/component/control_group_sortable_item.tsx b/src/plugins/presentation_util/public/components/controls/control_group/component/control_group_sortable_item.tsx index 3ae171a588da4..5c222e3c130b5 100644 --- a/src/plugins/presentation_util/public/components/controls/control_group/component/control_group_sortable_item.tsx +++ b/src/plugins/presentation_util/public/components/controls/control_group/component/control_group_sortable_item.tsx @@ -12,10 +12,9 @@ import { useSortable } from '@dnd-kit/sortable'; import { CSS } from '@dnd-kit/utilities'; import classNames from 'classnames'; -import { ControlWidth } from '../../types'; -import { ControlGroupContainer } from '../control_group_container'; -import { useChildEmbeddable } from '../../hooks/use_child_embeddable'; -import { ControlFrame, ControlFrameProps } from '../../control_frame/control_frame_component'; +import { ControlGroupInput } from '../types'; +import { ControlFrame, ControlFrameProps } from './control_frame_component'; +import { useReduxContainerContext } from '../../../redux_embeddables/redux_embeddable_context'; interface DragInfo { isOver?: boolean; @@ -26,7 +25,6 @@ interface DragInfo { export type SortableControlProps = ControlFrameProps & { dragInfo: DragInfo; - width: ControlWidth; }; /** @@ -60,91 +58,67 @@ export const SortableControl = (frameProps: SortableControlProps) => { const SortableControlInner = forwardRef< HTMLButtonElement, SortableControlProps & { style: HTMLAttributes['style'] } ->( - ( - { - embeddableId, - controlStyle, - container, - dragInfo, - onRemove, - onEdit, - style, - width, - ...dragHandleProps - }, - dragHandleRef - ) => { - const { isOver, isDragging, draggingIndex, index } = dragInfo; +>(({ embeddableId, dragInfo, style, ...dragHandleProps }, dragHandleRef) => { + const { isOver, isDragging, draggingIndex, index } = dragInfo; + const { useEmbeddableSelector } = useReduxContainerContext(); + const { panels } = useEmbeddableSelector((state) => state); - const dragHandle = ( - - ); + const width = panels[embeddableId].width; - return ( - (draggingIndex ?? -1), - })} - style={style} - > - - - ); - } -); + const dragHandle = ( + + ); + + return ( + (draggingIndex ?? -1), + })} + style={style} + > + + + ); +}); /** * A simplified clone version of the control which is dragged. This version only shows * the title, because individual controls can be any size, and dragging a wide item * can be quite cumbersome. */ -export const ControlClone = ({ - embeddableId, - container, - width, -}: { - embeddableId: string; - container: ControlGroupContainer; - width: ControlWidth; -}) => { - const embeddable = useChildEmbeddable({ embeddableId, container }); - const layout = container.getInput().controlStyle; +export const ControlClone = ({ draggingId }: { draggingId: string }) => { + const { useEmbeddableSelector } = useReduxContainerContext(); + const { panels, controlStyle } = useEmbeddableSelector((state) => state); + + const width = panels[draggingId].width; + const title = panels[draggingId].explicitInput.title; return ( - {layout === 'twoLine' ? ( - {embeddable?.getInput().title} - ) : undefined} + {controlStyle === 'twoLine' ? {title} : undefined} - {container.getInput().controlStyle === 'oneLine' ? ( - {embeddable?.getInput().title} - ) : undefined} + {controlStyle === 'oneLine' ? {title} : undefined} ); diff --git a/src/plugins/presentation_util/public/components/controls/control_group/control_group_container.tsx b/src/plugins/presentation_util/public/components/controls/control_group/control_group_container.tsx deleted file mode 100644 index 03249889dfdea..0000000000000 --- a/src/plugins/presentation_util/public/components/controls/control_group/control_group_container.tsx +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import React from 'react'; -import ReactDOM from 'react-dom'; -import { cloneDeep } from 'lodash'; - -import { - Container, - EmbeddableFactory, - EmbeddableFactoryNotFoundError, -} from '../../../../../embeddable/public'; -import { - InputControlEmbeddable, - InputControlInput, - InputControlOutput, - IEditableControlFactory, - ControlWidth, -} from '../types'; -import { ControlsService } from '../controls_service'; -import { ControlGroupInput, ControlPanelState } from './types'; -import { ManageControlComponent } from './editor/manage_control'; -import { toMountPoint } from '../../../../../kibana_react/public'; -import { ControlGroup } from './component/control_group_component'; -import { PresentationOverlaysService } from '../../../services/overlays'; -import { CONTROL_GROUP_TYPE, DEFAULT_CONTROL_WIDTH } from './control_group_constants'; -import { ManageControlGroup } from './editor/manage_control_group_component'; -import { OverlayRef } from '../../../../../../core/public'; -import { ControlGroupStrings } from './control_group_strings'; - -export class ControlGroupContainer extends Container { - public readonly type = CONTROL_GROUP_TYPE; - - private nextControlWidth: ControlWidth = DEFAULT_CONTROL_WIDTH; - - constructor( - initialInput: ControlGroupInput, - private readonly controlsService: ControlsService, - private readonly overlays: PresentationOverlaysService, - parent?: Container - ) { - super(initialInput, { embeddableLoaded: {} }, controlsService.getControlFactory, parent); - this.overlays = overlays; - this.controlsService = controlsService; - } - - protected createNewPanelState( - factory: EmbeddableFactory, - partial: Partial = {} - ): ControlPanelState { - const panelState = super.createNewPanelState(factory, partial); - return { - order: 1, - width: this.nextControlWidth, - ...panelState, - } as ControlPanelState; - } - - protected getInheritedInput(id: string): InputControlInput { - const { filters, query, timeRange, inheritParentState } = this.getInput(); - return { - filters: inheritParentState.useFilters ? filters : undefined, - query: inheritParentState.useQuery ? query : undefined, - timeRange: inheritParentState.useTimerange ? timeRange : undefined, - id, - }; - } - - public createNewControl = async (type: string) => { - const factory = this.controlsService.getControlFactory(type); - if (!factory) throw new EmbeddableFactoryNotFoundError(type); - - const initialInputPromise = new Promise>((resolve, reject) => { - let inputToReturn: Partial = {}; - - const onCancel = (ref: OverlayRef) => { - this.overlays - .openConfirm(ControlGroupStrings.management.discardNewControl.getSubtitle(), { - confirmButtonText: ControlGroupStrings.management.discardNewControl.getConfirm(), - cancelButtonText: ControlGroupStrings.management.discardNewControl.getCancel(), - title: ControlGroupStrings.management.discardNewControl.getTitle(), - buttonColor: 'danger', - }) - .then((confirmed) => { - if (confirmed) { - reject(); - ref.close(); - } - }); - }; - - const flyoutInstance = this.overlays.openFlyout( - toMountPoint( - (inputToReturn.title = newTitle)} - updateWidth={(newWidth) => (this.nextControlWidth = newWidth)} - controlEditorComponent={(factory as IEditableControlFactory).getControlEditor?.({ - onChange: (partialInput) => { - inputToReturn = { ...inputToReturn, ...partialInput }; - }, - })} - onSave={() => { - resolve(inputToReturn); - flyoutInstance.close(); - }} - onCancel={() => onCancel(flyoutInstance)} - /> - ), - { - onClose: (flyout) => onCancel(flyout), - } - ); - }); - initialInputPromise.then( - async (explicitInput) => { - await this.addNewEmbeddable(type, explicitInput); - }, - () => {} // swallow promise rejection because it can be part of normal flow - ); - }; - - public editControl = async (embeddableId: string) => { - const panel = this.getInput().panels[embeddableId]; - const factory = this.getFactory(panel.type); - const embeddable = await this.untilEmbeddableLoaded(embeddableId); - - if (!factory) throw new EmbeddableFactoryNotFoundError(panel.type); - - const initialExplicitInput = cloneDeep(panel.explicitInput); - const initialWidth = panel.width; - - const onCancel = (ref: OverlayRef) => { - this.overlays - .openConfirm(ControlGroupStrings.management.discardChanges.getSubtitle(), { - confirmButtonText: ControlGroupStrings.management.discardChanges.getConfirm(), - cancelButtonText: ControlGroupStrings.management.discardChanges.getCancel(), - title: ControlGroupStrings.management.discardChanges.getTitle(), - buttonColor: 'danger', - }) - .then((confirmed) => { - if (confirmed) { - embeddable.updateInput(initialExplicitInput); - this.updateInput({ - panels: { - ...this.getInput().panels, - [embeddableId]: { ...this.getInput().panels[embeddableId], width: initialWidth }, - }, - }); - ref.close(); - } - }); - }; - - const flyoutInstance = this.overlays.openFlyout( - toMountPoint( - this.removeEmbeddable(embeddableId)} - updateTitle={(newTitle) => embeddable.updateInput({ title: newTitle })} - controlEditorComponent={(factory as IEditableControlFactory).getControlEditor?.({ - onChange: (partialInput) => embeddable.updateInput(partialInput), - initialInput: embeddable.getInput(), - })} - onCancel={() => onCancel(flyoutInstance)} - onSave={() => flyoutInstance.close()} - updateWidth={(newWidth) => - this.updateInput({ - panels: { - ...this.getInput().panels, - [embeddableId]: { ...this.getInput().panels[embeddableId], width: newWidth }, - }, - }) - } - /> - ), - { - onClose: (flyout) => onCancel(flyout), - } - ); - }; - - public editControlGroup = () => { - const flyoutInstance = this.overlays.openFlyout( - toMountPoint( - this.updateInput({ controlStyle: newStyle })} - deleteAllEmbeddables={() => { - this.overlays - .openConfirm(ControlGroupStrings.management.deleteAllControls.getSubtitle(), { - confirmButtonText: ControlGroupStrings.management.deleteAllControls.getConfirm(), - cancelButtonText: ControlGroupStrings.management.deleteAllControls.getCancel(), - title: ControlGroupStrings.management.deleteAllControls.getTitle(), - buttonColor: 'danger', - }) - .then((confirmed) => { - if (confirmed) { - Object.keys(this.getInput().panels).forEach((id) => this.removeEmbeddable(id)); - flyoutInstance.close(); - } - }); - }} - setAllPanelWidths={(newWidth) => { - const newPanels = cloneDeep(this.getInput().panels); - Object.values(newPanels).forEach((panel) => (panel.width = newWidth)); - this.updateInput({ panels: { ...newPanels, ...newPanels } }); - }} - panels={this.getInput().panels} - /> - ) - ); - }; - - public render(dom: HTMLElement) { - ReactDOM.render(, dom); - } -} diff --git a/src/plugins/presentation_util/public/components/controls/control_group/control_group_strings.ts b/src/plugins/presentation_util/public/components/controls/control_group/control_group_strings.ts index 78e50d8651931..35e490b0ea530 100644 --- a/src/plugins/presentation_util/public/components/controls/control_group/control_group_strings.ts +++ b/src/plugins/presentation_util/public/components/controls/control_group/control_group_strings.ts @@ -48,13 +48,9 @@ export const ControlGroupStrings = { i18n.translate('presentationUtil.inputControls.controlGroup.management.flyoutTitle', { defaultMessage: 'Manage controls', }), - getDesignTitle: () => - i18n.translate('presentationUtil.inputControls.controlGroup.management.designTitle', { - defaultMessage: 'Design', - }), - getWidthTitle: () => - i18n.translate('presentationUtil.inputControls.controlGroup.management.widthTitle', { - defaultMessage: 'Width', + getDefaultWidthTitle: () => + i18n.translate('presentationUtil.inputControls.controlGroup.management.defaultWidthTitle', { + defaultMessage: 'Default width', }), getLayoutTitle: () => i18n.translate('presentationUtil.inputControls.controlGroup.management.layoutTitle', { @@ -64,23 +60,20 @@ export const ControlGroupStrings = { i18n.translate('presentationUtil.inputControls.controlGroup.management.delete', { defaultMessage: 'Delete control', }), + getSetAllWidthsToDefaultTitle: () => + i18n.translate('presentationUtil.inputControls.controlGroup.management.setAllWidths', { + defaultMessage: 'Set all widths to default', + }), getDeleteAllButtonTitle: () => i18n.translate('presentationUtil.inputControls.controlGroup.management.deleteAll', { defaultMessage: 'Delete all', }), controlWidth: { - getChangeAllControlWidthsTitle: () => - i18n.translate( - 'presentationUtil.inputControls.controlGroup.management.layout.changeAllControlWidths', - { - defaultMessage: 'Set width for all controls', - } - ), getWidthSwitchLegend: () => i18n.translate( 'presentationUtil.inputControls.controlGroup.management.layout.controlWidthLegend', { - defaultMessage: 'Change individual control width', + defaultMessage: 'Change control width', } ), getAutoWidthTitle: () => @@ -117,21 +110,31 @@ export const ControlGroupStrings = { defaultMessage: 'Two line layout', }), }, - deleteAllControls: { - getTitle: () => - i18n.translate('presentationUtil.inputControls.controlGroup.management.deleteAll.title', { - defaultMessage: 'Delete all?', - }), + deleteControls: { + getDeleteAllTitle: () => + i18n.translate( + 'presentationUtil.inputControls.controlGroup.management.delete.deleteAllTitle', + { + defaultMessage: 'Delete all controls?', + } + ), + getDeleteTitle: () => + i18n.translate( + 'presentationUtil.inputControls.controlGroup.management.delete.deleteTitle', + { + defaultMessage: 'Delete control?', + } + ), getSubtitle: () => - i18n.translate('presentationUtil.inputControls.controlGroup.management.deleteAll.sub', { + i18n.translate('presentationUtil.inputControls.controlGroup.management.delete.sub', { defaultMessage: 'Controls are not recoverable once removed.', }), getConfirm: () => - i18n.translate('presentationUtil.inputControls.controlGroup.management.deleteAll.confirm', { + i18n.translate('presentationUtil.inputControls.controlGroup.management.delete.confirm', { defaultMessage: 'Delete', }), getCancel: () => - i18n.translate('presentationUtil.inputControls.controlGroup.management.deleteAll.cancel', { + i18n.translate('presentationUtil.inputControls.controlGroup.management.delete.cancel', { defaultMessage: 'Cancel', }), }, @@ -143,7 +146,7 @@ export const ControlGroupStrings = { getSubtitle: () => i18n.translate('presentationUtil.inputControls.controlGroup.management.discard.sub', { defaultMessage: - 'Discard changes to this control? Controls are not recoverable once removed.', + 'Discard changes to this control? Changes are not recoverable once discardsd.', }), getConfirm: () => i18n.translate('presentationUtil.inputControls.controlGroup.management.discard.confirm', { @@ -161,7 +164,7 @@ export const ControlGroupStrings = { }), getSubtitle: () => i18n.translate('presentationUtil.inputControls.controlGroup.management.deleteNew.sub', { - defaultMessage: 'Discard new control? Controls are not recoverable once removed.', + defaultMessage: 'Discard new control? Controls are not recoverable once discarded.', }), getConfirm: () => i18n.translate('presentationUtil.inputControls.controlGroup.management.deleteNew.confirm', { @@ -173,4 +176,14 @@ export const ControlGroupStrings = { }), }, }, + floatingActions: { + getEditButtonTitle: () => + i18n.translate('presentationUtil.inputControls.controlGroup.floatingActions.editTitle', { + defaultMessage: 'Manage control', + }), + getRemoveButtonTitle: () => + i18n.translate('presentationUtil.inputControls.controlGroup.floatingActions.removeTitle', { + defaultMessage: 'Remove control', + }), + }, }; diff --git a/src/plugins/presentation_util/public/components/controls/control_group/editor/manage_control.tsx b/src/plugins/presentation_util/public/components/controls/control_group/editor/control_editor.tsx similarity index 99% rename from src/plugins/presentation_util/public/components/controls/control_group/editor/manage_control.tsx rename to src/plugins/presentation_util/public/components/controls/control_group/editor/control_editor.tsx index 6d80a6e0b31f6..38d8faf37397a 100644 --- a/src/plugins/presentation_util/public/components/controls/control_group/editor/manage_control.tsx +++ b/src/plugins/presentation_util/public/components/controls/control_group/editor/control_editor.tsx @@ -46,7 +46,7 @@ interface ManageControlProps { updateWidth: (newWidth: ControlWidth) => void; } -export const ManageControlComponent = ({ +export const ControlEditor = ({ controlEditorComponent, removeControl, updateTitle, diff --git a/src/plugins/presentation_util/public/components/controls/control_group/editor/create_control.tsx b/src/plugins/presentation_util/public/components/controls/control_group/editor/create_control.tsx new file mode 100644 index 0000000000000..9f59fe98cc0c1 --- /dev/null +++ b/src/plugins/presentation_util/public/components/controls/control_group/editor/create_control.tsx @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + EuiButtonIcon, + EuiButtonIconColor, + EuiContextMenuItem, + EuiContextMenuPanel, + EuiPopover, +} from '@elastic/eui'; +import React, { useState, ReactElement } from 'react'; + +import { ControlGroupInput } from '../types'; +import { ControlEditor } from './control_editor'; +import { pluginServices } from '../../../../services'; +import { forwardAllContext } from './forward_all_context'; +import { OverlayRef } from '../../../../../../../core/public'; +import { ControlGroupStrings } from '../control_group_strings'; +import { InputControlInput } from '../../../../services/controls'; +import { DEFAULT_CONTROL_WIDTH } from '../control_group_constants'; +import { ControlWidth, IEditableControlFactory } from '../../types'; +import { controlGroupReducers } from '../state/control_group_reducers'; +import { EmbeddableFactoryNotFoundError } from '../../../../../../embeddable/public'; +import { useReduxContainerContext } from '../../../redux_embeddables/redux_embeddable_context'; + +export const CreateControlButton = () => { + // Presentation Services Context + const { overlays, controls } = pluginServices.getHooks(); + const { getInputControlTypes, getControlFactory } = controls.useService(); + const { openFlyout, openConfirm } = overlays.useService(); + + // Redux embeddable container Context + const reduxContainerContext = useReduxContainerContext< + ControlGroupInput, + typeof controlGroupReducers + >(); + const { + containerActions: { addNewEmbeddable }, + actions: { setDefaultControlWidth }, + useEmbeddableSelector, + useEmbeddableDispatch, + } = reduxContainerContext; + const dispatch = useEmbeddableDispatch(); + + // current state + const { defaultControlWidth } = useEmbeddableSelector((state) => state); + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + + const createNewControl = async (type: string) => { + const factory = getControlFactory(type); + if (!factory) throw new EmbeddableFactoryNotFoundError(type); + + const initialInputPromise = new Promise>((resolve, reject) => { + let inputToReturn: Partial = {}; + + const onCancel = (ref: OverlayRef) => { + if (Object.keys(inputToReturn).length === 0) { + reject(); + ref.close(); + return; + } + openConfirm(ControlGroupStrings.management.discardNewControl.getSubtitle(), { + confirmButtonText: ControlGroupStrings.management.discardNewControl.getConfirm(), + cancelButtonText: ControlGroupStrings.management.discardNewControl.getCancel(), + title: ControlGroupStrings.management.discardNewControl.getTitle(), + buttonColor: 'danger', + }).then((confirmed) => { + if (confirmed) { + reject(); + ref.close(); + } + }); + }; + + const flyoutInstance = openFlyout( + forwardAllContext( + (inputToReturn.title = newTitle)} + updateWidth={(newWidth) => dispatch(setDefaultControlWidth(newWidth as ControlWidth))} + controlEditorComponent={(factory as IEditableControlFactory).getControlEditor?.({ + onChange: (partialInput) => { + inputToReturn = { ...inputToReturn, ...partialInput }; + }, + })} + onSave={() => { + resolve(inputToReturn); + flyoutInstance.close(); + }} + onCancel={() => onCancel(flyoutInstance)} + />, + reduxContainerContext + ), + { + onClose: (flyout) => onCancel(flyout), + } + ); + }); + initialInputPromise.then( + async (explicitInput) => { + await addNewEmbeddable(type, explicitInput); + }, + () => {} // swallow promise rejection because it can be part of normal flow + ); + }; + + if (getInputControlTypes().length === 0) return null; + + const commonButtonProps = { + iconType: 'plus', + color: 'text' as EuiButtonIconColor, + 'data-test-subj': 'inputControlsSortingButton', + 'aria-label': ControlGroupStrings.management.getManageButtonTitle(), + }; + + if (getInputControlTypes().length > 1) { + const items: ReactElement[] = []; + getInputControlTypes().forEach((type) => { + const factory = getControlFactory(type); + items.push( + { + setIsPopoverOpen(false); + createNewControl(type); + }} + > + {factory.getDisplayName()} + + ); + }); + const button = setIsPopoverOpen(true)} />; + + return ( + setIsPopoverOpen(false)} + > + + + ); + } + return ( + createNewControl(getInputControlTypes()[0])} + /> + ); +}; diff --git a/src/plugins/presentation_util/public/components/controls/control_group/editor/edit_control.tsx b/src/plugins/presentation_util/public/components/controls/control_group/editor/edit_control.tsx new file mode 100644 index 0000000000000..58c59c8f84fe0 --- /dev/null +++ b/src/plugins/presentation_util/public/components/controls/control_group/editor/edit_control.tsx @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { isEqual } from 'lodash'; +import { EuiButtonIcon } from '@elastic/eui'; +import React, { useEffect, useRef } from 'react'; + +import { ControlGroupInput } from '../types'; +import { ControlEditor } from './control_editor'; +import { IEditableControlFactory } from '../../types'; +import { pluginServices } from '../../../../services'; +import { forwardAllContext } from './forward_all_context'; +import { OverlayRef } from '../../../../../../../core/public'; +import { ControlGroupStrings } from '../control_group_strings'; +import { controlGroupReducers } from '../state/control_group_reducers'; +import { EmbeddableFactoryNotFoundError } from '../../../../../../embeddable/public'; +import { useReduxContainerContext } from '../../../redux_embeddables/redux_embeddable_context'; + +export const EditControlButton = ({ embeddableId }: { embeddableId: string }) => { + // Presentation Services Context + const { overlays, controls } = pluginServices.getHooks(); + const { getControlFactory } = controls.useService(); + const { openFlyout, openConfirm } = overlays.useService(); + + // Redux embeddable container Context + const reduxContainerContext = useReduxContainerContext< + ControlGroupInput, + typeof controlGroupReducers + >(); + const { + containerActions: { untilEmbeddableLoaded, removeEmbeddable, updateInputForChild }, + actions: { setControlWidth }, + useEmbeddableSelector, + useEmbeddableDispatch, + } = reduxContainerContext; + const dispatch = useEmbeddableDispatch(); + + // current state + const { panels } = useEmbeddableSelector((state) => state); + + // keep up to date ref of latest panel state for comparison when closing editor. + const latestPanelState = useRef(panels[embeddableId]); + useEffect(() => { + latestPanelState.current = panels[embeddableId]; + }, [panels, embeddableId]); + + const editControl = async () => { + const panel = panels[embeddableId]; + const factory = getControlFactory(panel.type); + const embeddable = await untilEmbeddableLoaded(embeddableId); + + if (!factory) throw new EmbeddableFactoryNotFoundError(panel.type); + + let removed = false; + const onCancel = (ref: OverlayRef) => { + if ( + removed || + (isEqual(latestPanelState.current.explicitInput, panel.explicitInput) && + isEqual(latestPanelState.current.width, panel.width)) + ) { + ref.close(); + return; + } + openConfirm(ControlGroupStrings.management.discardChanges.getSubtitle(), { + confirmButtonText: ControlGroupStrings.management.discardChanges.getConfirm(), + cancelButtonText: ControlGroupStrings.management.discardChanges.getCancel(), + title: ControlGroupStrings.management.discardChanges.getTitle(), + buttonColor: 'danger', + }).then((confirmed) => { + if (confirmed) { + updateInputForChild(embeddableId, panel.explicitInput); + dispatch(setControlWidth({ width: panel.width, embeddableId })); + ref.close(); + } + }); + }; + + const flyoutInstance = openFlyout( + forwardAllContext( + { + openConfirm(ControlGroupStrings.management.deleteControls.getSubtitle(), { + confirmButtonText: ControlGroupStrings.management.deleteControls.getConfirm(), + cancelButtonText: ControlGroupStrings.management.deleteControls.getCancel(), + title: ControlGroupStrings.management.deleteControls.getDeleteTitle(), + buttonColor: 'danger', + }).then((confirmed) => { + if (confirmed) { + removeEmbeddable(embeddableId); + removed = true; + flyoutInstance.close(); + } + }); + }} + updateTitle={(newTitle) => updateInputForChild(embeddableId, { title: newTitle })} + controlEditorComponent={(factory as IEditableControlFactory).getControlEditor?.({ + onChange: (partialInput) => updateInputForChild(embeddableId, partialInput), + initialInput: embeddable.getInput(), + })} + onCancel={() => onCancel(flyoutInstance)} + onSave={() => flyoutInstance.close()} + updateWidth={(newWidth) => dispatch(setControlWidth({ width: newWidth, embeddableId }))} + />, + reduxContainerContext + ), + { + onClose: (flyout) => onCancel(flyout), + } + ); + }; + + return ( + editControl()} + color="text" + /> + ); +}; diff --git a/src/plugins/presentation_util/public/components/controls/control_group/editor/edit_control_group.tsx b/src/plugins/presentation_util/public/components/controls/control_group/editor/edit_control_group.tsx new file mode 100644 index 0000000000000..9438091e2fb1d --- /dev/null +++ b/src/plugins/presentation_util/public/components/controls/control_group/editor/edit_control_group.tsx @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React from 'react'; +import { + EuiTitle, + EuiSpacer, + EuiFormRow, + EuiFlexItem, + EuiFlexGroup, + EuiFlyoutBody, + EuiButtonGroup, + EuiButtonEmpty, + EuiFlyoutHeader, +} from '@elastic/eui'; + +import { + CONTROL_LAYOUT_OPTIONS, + CONTROL_WIDTH_OPTIONS, + DEFAULT_CONTROL_WIDTH, +} from '../control_group_constants'; +import { ControlGroupInput } from '../types'; +import { pluginServices } from '../../../../services'; +import { ControlStyle, ControlWidth } from '../../types'; +import { ControlGroupStrings } from '../control_group_strings'; +import { controlGroupReducers } from '../state/control_group_reducers'; +import { useReduxContainerContext } from '../../../redux_embeddables/redux_embeddable_context'; + +export const EditControlGroup = () => { + const { overlays } = pluginServices.getHooks(); + const { openConfirm } = overlays.useService(); + + const { + containerActions, + useEmbeddableSelector, + useEmbeddableDispatch, + actions: { setControlStyle, setAllControlWidths, setDefaultControlWidth }, + } = useReduxContainerContext(); + + const dispatch = useEmbeddableDispatch(); + const { panels, controlStyle, defaultControlWidth } = useEmbeddableSelector((state) => state); + + return ( + <> + + +

{ControlGroupStrings.management.getFlyoutTitle()}

+ + + + + + dispatch(setControlStyle(newControlStyle as ControlStyle)) + } + /> + + + + + + + dispatch(setDefaultControlWidth(newWidth as ControlWidth)) + } + /> + + + + dispatch(setAllControlWidths(defaultControlWidth ?? DEFAULT_CONTROL_WIDTH)) + } + aria-label={'delete-all'} + iconType="returnKey" + size="s" + > + {ControlGroupStrings.management.getSetAllWidthsToDefaultTitle()} + + + + + + + + { + if (!containerActions?.removeEmbeddable) return; + openConfirm(ControlGroupStrings.management.deleteControls.getSubtitle(), { + confirmButtonText: ControlGroupStrings.management.deleteControls.getConfirm(), + cancelButtonText: ControlGroupStrings.management.deleteControls.getCancel(), + title: ControlGroupStrings.management.deleteControls.getDeleteAllTitle(), + buttonColor: 'danger', + }).then((confirmed) => { + if (confirmed) + Object.keys(panels).forEach((panelId) => + containerActions.removeEmbeddable(panelId) + ); + }); + }} + aria-label={'delete-all'} + iconType="trash" + color="danger" + flush="left" + size="s" + > + {ControlGroupStrings.management.getDeleteAllButtonTitle()} + + + + ); +}; diff --git a/src/plugins/presentation_util/public/components/controls/control_group/editor/forward_all_context.tsx b/src/plugins/presentation_util/public/components/controls/control_group/editor/forward_all_context.tsx new file mode 100644 index 0000000000000..bb7356c240648 --- /dev/null +++ b/src/plugins/presentation_util/public/components/controls/control_group/editor/forward_all_context.tsx @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Provider } from 'react-redux'; +import { ReactElement } from 'react'; +import React from 'react'; + +import { ControlGroupInput } from '../types'; +import { pluginServices } from '../../../../services'; +import { toMountPoint } from '../../../../../../kibana_react/public'; +import { ReduxContainerContextServices } from '../../../redux_embeddables/types'; +import { ReduxEmbeddableContext } from '../../../redux_embeddables/redux_embeddable_context'; +import { getManagedEmbeddablesStore } from '../../../redux_embeddables/generic_embeddable_store'; + +/** + * The overlays service creates its divs outside the flow of the component. This necessitates + * passing all context from the component to the flyout. + */ +export const forwardAllContext = ( + component: ReactElement, + reduxContainerContext: ReduxContainerContextServices +) => { + const PresentationUtilProvider = pluginServices.getContextProvider(); + return toMountPoint( + + + {component} + + + ); +}; diff --git a/src/plugins/presentation_util/public/components/controls/control_group/editor/manage_control_group_component.tsx b/src/plugins/presentation_util/public/components/controls/control_group/editor/manage_control_group_component.tsx deleted file mode 100644 index e766b16ade13a..0000000000000 --- a/src/plugins/presentation_util/public/components/controls/control_group/editor/manage_control_group_component.tsx +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import useMount from 'react-use/lib/useMount'; -import React, { useState } from 'react'; -import { - EuiFlyoutHeader, - EuiButtonEmpty, - EuiButtonGroup, - EuiFlyoutBody, - EuiFormRow, - EuiSpacer, - EuiSwitch, - EuiTitle, -} from '@elastic/eui'; - -import { ControlsPanels } from '../types'; -import { ControlStyle, ControlWidth } from '../../types'; -import { ControlGroupStrings } from '../control_group_strings'; -import { CONTROL_LAYOUT_OPTIONS, CONTROL_WIDTH_OPTIONS } from '../control_group_constants'; - -interface ManageControlGroupProps { - panels: ControlsPanels; - controlStyle: ControlStyle; - deleteAllEmbeddables: () => void; - setControlStyle: (style: ControlStyle) => void; - setAllPanelWidths: (newWidth: ControlWidth) => void; -} - -export const ManageControlGroup = ({ - panels, - controlStyle, - setControlStyle, - setAllPanelWidths, - deleteAllEmbeddables, -}: ManageControlGroupProps) => { - const [currentControlStyle, setCurrentControlStyle] = useState(controlStyle); - const [selectedWidth, setSelectedWidth] = useState(); - const [selectionDisplay, setSelectionDisplay] = useState(false); - - useMount(() => { - if (!panels || Object.keys(panels).length === 0) return; - const firstWidth = panels[Object.keys(panels)[0]].width; - if (Object.values(panels).every((panel) => panel.width === firstWidth)) { - setSelectedWidth(firstWidth); - } - }); - - return ( - <> - - -

{ControlGroupStrings.management.getFlyoutTitle()}

- - - - - { - setControlStyle(newControlStyle as ControlStyle); - setCurrentControlStyle(newControlStyle as ControlStyle); - }} - /> - - - - setSelectionDisplay(!selectionDisplay)} - /> - - {selectionDisplay ? ( - <> - - { - setAllPanelWidths(newWidth as ControlWidth); - setSelectedWidth(newWidth as ControlWidth); - }} - /> - - ) : undefined} - - - - - {ControlGroupStrings.management.getDeleteAllButtonTitle()} - - - - ); -}; diff --git a/src/plugins/presentation_util/public/components/controls/control_group/embeddable/control_group_container.tsx b/src/plugins/presentation_util/public/components/controls/control_group/embeddable/control_group_container.tsx new file mode 100644 index 0000000000000..a722bed6c07d2 --- /dev/null +++ b/src/plugins/presentation_util/public/components/controls/control_group/embeddable/control_group_container.tsx @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React from 'react'; +import ReactDOM from 'react-dom'; + +import { + InputControlEmbeddable, + InputControlInput, + InputControlOutput, +} from '../../../../services/controls'; +import { pluginServices } from '../../../../services'; +import { ControlGroupInput, ControlPanelState } from '../types'; +import { ControlGroup } from '../component/control_group_component'; +import { controlGroupReducers } from '../state/control_group_reducers'; +import { Container, EmbeddableFactory } from '../../../../../../embeddable/public'; +import { CONTROL_GROUP_TYPE, DEFAULT_CONTROL_WIDTH } from '../control_group_constants'; +import { ReduxEmbeddableWrapper } from '../../../redux_embeddables/redux_embeddable_wrapper'; + +export class ControlGroupContainer extends Container { + public readonly type = CONTROL_GROUP_TYPE; + + constructor(initialInput: ControlGroupInput, parent?: Container) { + super( + initialInput, + { embeddableLoaded: {} }, + pluginServices.getServices().controls.getControlFactory, + parent + ); + } + + protected createNewPanelState( + factory: EmbeddableFactory, + partial: Partial = {} + ): ControlPanelState { + const panelState = super.createNewPanelState(factory, partial); + const highestOrder = Object.values(this.getInput().panels).reduce((highestSoFar, panel) => { + if (panel.order > highestSoFar) highestSoFar = panel.order; + return highestSoFar; + }, 0); + return { + order: highestOrder + 1, + width: this.getInput().defaultControlWidth ?? DEFAULT_CONTROL_WIDTH, + ...panelState, + } as ControlPanelState; + } + + protected getInheritedInput(id: string): InputControlInput { + const { filters, query, timeRange, inheritParentState } = this.getInput(); + return { + filters: inheritParentState.useFilters ? filters : undefined, + query: inheritParentState.useQuery ? query : undefined, + timeRange: inheritParentState.useTimerange ? timeRange : undefined, + id, + }; + } + + public render(dom: HTMLElement) { + const PresentationUtilProvider = pluginServices.getContextProvider(); + ReactDOM.render( + + + embeddable={this} + reducers={controlGroupReducers} + > + + + , + dom + ); + } +} diff --git a/src/plugins/presentation_util/public/components/controls/control_group/control_group_container_factory.ts b/src/plugins/presentation_util/public/components/controls/control_group/embeddable/control_group_container_factory.ts similarity index 71% rename from src/plugins/presentation_util/public/components/controls/control_group/control_group_container_factory.ts rename to src/plugins/presentation_util/public/components/controls/control_group/embeddable/control_group_container_factory.ts index 97ef48e6b240c..e50b1c5d734e4 100644 --- a/src/plugins/presentation_util/public/components/controls/control_group/control_group_container_factory.ts +++ b/src/plugins/presentation_util/public/components/controls/control_group/embeddable/control_group_container_factory.ts @@ -20,13 +20,11 @@ import { EmbeddableFactory, EmbeddableFactoryDefinition, ErrorEmbeddable, -} from '../../../../../embeddable/public'; -import { ControlGroupInput } from './types'; -import { ControlsService } from '../controls_service'; -import { ControlGroupStrings } from './control_group_strings'; -import { CONTROL_GROUP_TYPE } from './control_group_constants'; +} from '../../../../../../embeddable/public'; +import { ControlGroupInput } from '../types'; +import { ControlGroupStrings } from '../control_group_strings'; +import { CONTROL_GROUP_TYPE } from '../control_group_constants'; import { ControlGroupContainer } from './control_group_container'; -import { PresentationOverlaysService } from '../../../services/overlays'; export type DashboardContainerFactory = EmbeddableFactory< ControlGroupInput, @@ -38,13 +36,6 @@ export class ControlGroupContainerFactory { public readonly isContainerType = true; public readonly type = CONTROL_GROUP_TYPE; - public readonly controlsService: ControlsService; - private readonly overlays: PresentationOverlaysService; - - constructor(controlsService: ControlsService, overlays: PresentationOverlaysService) { - this.overlays = overlays; - this.controlsService = controlsService; - } public isEditable = async () => false; @@ -67,6 +58,6 @@ export class ControlGroupContainerFactory initialInput: ControlGroupInput, parent?: Container ): Promise => { - return new ControlGroupContainer(initialInput, this.controlsService, this.overlays, parent); + return new ControlGroupContainer(initialInput, parent); }; } diff --git a/src/plugins/presentation_util/public/components/controls/control_group/state/control_group_reducers.ts b/src/plugins/presentation_util/public/components/controls/control_group/state/control_group_reducers.ts new file mode 100644 index 0000000000000..b7c0c62535d4c --- /dev/null +++ b/src/plugins/presentation_util/public/components/controls/control_group/state/control_group_reducers.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PayloadAction } from '@reduxjs/toolkit'; +import { WritableDraft } from 'immer/dist/types/types-external'; + +import { ControlWidth } from '../../types'; +import { ControlGroupInput } from '../types'; + +export const controlGroupReducers = { + setControlStyle: ( + state: WritableDraft, + action: PayloadAction + ) => { + state.controlStyle = action.payload; + }, + setDefaultControlWidth: ( + state: WritableDraft, + action: PayloadAction + ) => { + state.defaultControlWidth = action.payload; + }, + setAllControlWidths: ( + state: WritableDraft, + action: PayloadAction + ) => { + Object.keys(state.panels).forEach((panelId) => (state.panels[panelId].width = action.payload)); + }, + setControlWidth: ( + state: WritableDraft, + action: PayloadAction<{ width: ControlWidth; embeddableId: string }> + ) => { + state.panels[action.payload.embeddableId].width = action.payload.width; + }, + setControlOrders: ( + state: WritableDraft, + action: PayloadAction<{ ids: string[] }> + ) => { + action.payload.ids.forEach((id, index) => { + state.panels[id].order = index; + }); + }, +}; diff --git a/src/plugins/presentation_util/public/components/controls/control_group/types.ts b/src/plugins/presentation_util/public/components/controls/control_group/types.ts index fb381610711e5..438eee1c461dd 100644 --- a/src/plugins/presentation_util/public/components/controls/control_group/types.ts +++ b/src/plugins/presentation_util/public/components/controls/control_group/types.ts @@ -7,7 +7,8 @@ */ import { PanelState, EmbeddableInput } from '../../../../../embeddable/public'; -import { ControlStyle, ControlWidth, InputControlInput } from '../types'; +import { InputControlInput } from '../../../services/controls'; +import { ControlStyle, ControlWidth } from '../types'; export interface ControlGroupInput extends EmbeddableInput, @@ -17,6 +18,7 @@ export interface ControlGroupInput useQuery: boolean; useTimerange: boolean; }; + defaultControlWidth?: ControlWidth; controlStyle: ControlStyle; panels: ControlsPanels; } diff --git a/src/plugins/presentation_util/public/components/controls/control_types/options_list/options_list_embeddable.tsx b/src/plugins/presentation_util/public/components/controls/control_types/options_list/options_list_embeddable.tsx index 93a7b3e353bdf..97a128c3e84eb 100644 --- a/src/plugins/presentation_util/public/components/controls/control_types/options_list/options_list_embeddable.tsx +++ b/src/plugins/presentation_util/public/components/controls/control_types/options_list/options_list_embeddable.tsx @@ -16,7 +16,7 @@ import { tap, debounceTime, map, distinctUntilChanged } from 'rxjs/operators'; import { esFilters } from '../../../../../../data/public'; import { OptionsListStrings } from './options_list_strings'; import { Embeddable, IContainer } from '../../../../../../embeddable/public'; -import { InputControlInput, InputControlOutput } from '../../types'; +import { InputControlInput, InputControlOutput } from '../../../../services/controls'; import { OptionsListComponent, OptionsListComponentState } from './options_list_component'; const toggleAvailableOptions = ( diff --git a/src/plugins/presentation_util/public/components/controls/controls_service.ts b/src/plugins/presentation_util/public/components/controls/controls_service.ts index 4e01f3cf9ab6a..82242946e4563 100644 --- a/src/plugins/presentation_util/public/components/controls/controls_service.ts +++ b/src/plugins/presentation_util/public/components/controls/controls_service.ts @@ -8,12 +8,12 @@ import { EmbeddableFactory } from '../../../../embeddable/public'; import { - ControlTypeRegistry, InputControlEmbeddable, + ControlTypeRegistry, InputControlFactory, - InputControlInput, InputControlOutput, -} from './types'; + InputControlInput, +} from '../../services/controls'; export class ControlsService { private controlsFactoriesMap: ControlTypeRegistry = {}; diff --git a/src/plugins/presentation_util/public/components/controls/hooks/use_child_embeddable.ts b/src/plugins/presentation_util/public/components/controls/hooks/use_child_embeddable.ts index 82b9aa528bf35..c4f700ec059d9 100644 --- a/src/plugins/presentation_util/public/components/controls/hooks/use_child_embeddable.ts +++ b/src/plugins/presentation_util/public/components/controls/hooks/use_child_embeddable.ts @@ -6,14 +6,13 @@ * Side Public License, v 1. */ import { useEffect, useState } from 'react'; -import { InputControlEmbeddable } from '../types'; -import { IContainer } from '../../../../../embeddable/public'; +import { InputControlEmbeddable } from '../../../services/controls'; export const useChildEmbeddable = ({ - container, + untilEmbeddableLoaded, embeddableId, }: { - container: IContainer; + untilEmbeddableLoaded: (embeddableId: string) => Promise; embeddableId: string; }) => { const [embeddable, setEmbeddable] = useState(); @@ -21,14 +20,14 @@ export const useChildEmbeddable = ({ useEffect(() => { let mounted = true; (async () => { - const newEmbeddable = await container.untilEmbeddableLoaded(embeddableId); + const newEmbeddable = await untilEmbeddableLoaded(embeddableId); if (!mounted) return; setEmbeddable(newEmbeddable); })(); return () => { mounted = false; }; - }, [container, embeddableId]); + }, [untilEmbeddableLoaded, embeddableId]); return embeddable; }; diff --git a/src/plugins/presentation_util/public/components/controls/types.ts b/src/plugins/presentation_util/public/components/controls/types.ts index c94e2957e34ea..0704a601640e6 100644 --- a/src/plugins/presentation_util/public/components/controls/types.ts +++ b/src/plugins/presentation_util/public/components/controls/types.ts @@ -6,47 +6,11 @@ * Side Public License, v 1. */ -import { Filter } from '@kbn/es-query'; -import { Query, TimeRange } from '../../../../data/public'; -import { - EmbeddableFactory, - EmbeddableInput, - EmbeddableOutput, - IEmbeddable, -} from '../../../../embeddable/public'; +import { InputControlInput } from '../../services/controls'; export type ControlWidth = 'auto' | 'small' | 'medium' | 'large'; export type ControlStyle = 'twoLine' | 'oneLine'; -/** - * Control embeddable types - */ -export type InputControlFactory = EmbeddableFactory< - InputControlInput, - InputControlOutput, - InputControlEmbeddable ->; - -export interface ControlTypeRegistry { - [key: string]: InputControlFactory; -} - -export type InputControlInput = EmbeddableInput & { - query?: Query; - filters?: Filter[]; - timeRange?: TimeRange; - twoLineLayout?: boolean; -}; - -export type InputControlOutput = EmbeddableOutput & { - filters?: Filter[]; -}; - -export type InputControlEmbeddable< - TInputControlEmbeddableInput extends InputControlInput = InputControlInput, - TInputControlEmbeddableOutput extends InputControlOutput = InputControlOutput -> = IEmbeddable; - /** * Control embeddable editor types */ diff --git a/src/plugins/presentation_util/public/components/redux_embeddables/generic_embeddable_store.ts b/src/plugins/presentation_util/public/components/redux_embeddables/generic_embeddable_store.ts new file mode 100644 index 0000000000000..36ba1fcaa49b9 --- /dev/null +++ b/src/plugins/presentation_util/public/components/redux_embeddables/generic_embeddable_store.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { configureStore, EnhancedStore } from '@reduxjs/toolkit'; +import { combineReducers, Reducer } from 'redux'; + +export interface InjectReducerProps { + key: string; + asyncReducer: Reducer; +} + +type ManagedEmbeddableReduxStore = EnhancedStore & { + asyncReducers: { [key: string]: Reducer }; + injectReducer: (props: InjectReducerProps) => void; +}; +const embeddablesStore = configureStore({ reducer: {} as { [key: string]: Reducer } }); + +const managedEmbeddablesStore = embeddablesStore as ManagedEmbeddableReduxStore; +managedEmbeddablesStore.asyncReducers = {}; + +managedEmbeddablesStore.injectReducer = ({ + key, + asyncReducer, +}: InjectReducerProps) => { + managedEmbeddablesStore.asyncReducers[key] = asyncReducer as Reducer; + managedEmbeddablesStore.replaceReducer( + combineReducers({ ...managedEmbeddablesStore.asyncReducers }) + ); +}; + +/** + * A managed Redux store which can be used with multiple embeddables at once. When a new embeddable is created at runtime, + * all passed in reducers will be made into a slice, then combined into the store using combineReducers. + */ +export const getManagedEmbeddablesStore = () => managedEmbeddablesStore; diff --git a/src/plugins/presentation_util/public/components/redux_embeddables/redux_embeddable_context.ts b/src/plugins/presentation_util/public/components/redux_embeddables/redux_embeddable_context.ts new file mode 100644 index 0000000000000..159230e4de024 --- /dev/null +++ b/src/plugins/presentation_util/public/components/redux_embeddables/redux_embeddable_context.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { createContext, useContext } from 'react'; + +import { + GenericEmbeddableReducers, + ReduxContainerContextServices, + ReduxEmbeddableContextServices, +} from './types'; +import { ContainerInput, EmbeddableInput } from '../../../../embeddable/public'; + +/** + * When creating the context, a generic EmbeddableInput as placeholder is used. This will later be cast to + * the generic type passed in by the useReduxEmbeddableContext or useReduxContainerContext hooks + **/ +export const ReduxEmbeddableContext = createContext< + | ReduxEmbeddableContextServices + | ReduxContainerContextServices + | null +>(null); + +/** + * A typed use context hook for embeddables that are not containers. it @returns an + * ReduxEmbeddableContextServices object typed to the generic inputTypes and ReducerTypes you pass in. + * Note that the reducer type is optional, but will be required to correctly infer the keys and payload + * types of your reducers. use `typeof MyReducers` here to retain them. + */ +export const useReduxEmbeddableContext = < + InputType extends EmbeddableInput = EmbeddableInput, + ReducerType extends GenericEmbeddableReducers = GenericEmbeddableReducers +>(): ReduxEmbeddableContextServices => { + const context = useContext>( + ReduxEmbeddableContext as unknown as React.Context< + ReduxEmbeddableContextServices + > + ); + if (context == null) { + throw new Error( + 'useReduxEmbeddableContext must be used inside the useReduxEmbeddableContextProvider.' + ); + } + + return context!; +}; + +/** + * A typed use context hook for embeddable containers. it @returns an + * ReduxContainerContextServices object typed to the generic inputTypes and ReducerTypes you pass in. + * Note that the reducer type is optional, but will be required to correctly infer the keys and payload + * types of your reducers. use `typeof MyReducers` here to retain them. It also includes a containerActions + * key which contains most of the commonly used container operations + */ +export const useReduxContainerContext = < + InputType extends ContainerInput = ContainerInput, + ReducerType extends GenericEmbeddableReducers = GenericEmbeddableReducers +>(): ReduxContainerContextServices => { + const context = useContext>( + ReduxEmbeddableContext as unknown as React.Context< + ReduxContainerContextServices + > + ); + if (context == null) { + throw new Error( + 'useReduxEmbeddableContext must be used inside the useReduxEmbeddableContextProvider.' + ); + } + return context!; +}; diff --git a/src/plugins/presentation_util/public/components/redux_embeddables/redux_embeddable_wrapper.tsx b/src/plugins/presentation_util/public/components/redux_embeddables/redux_embeddable_wrapper.tsx new file mode 100644 index 0000000000000..a4912b5b5f2fc --- /dev/null +++ b/src/plugins/presentation_util/public/components/redux_embeddables/redux_embeddable_wrapper.tsx @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React, { PropsWithChildren, useEffect, useMemo, useRef } from 'react'; +import { Provider, TypedUseSelectorHook, useDispatch, useSelector } from 'react-redux'; +import { Draft } from 'immer/dist/types/types-external'; +import { isEqual } from 'lodash'; +import { SliceCaseReducers, PayloadAction, createSlice } from '@reduxjs/toolkit'; + +import { + IEmbeddable, + EmbeddableInput, + EmbeddableOutput, + IContainer, +} from '../../../../embeddable/public'; +import { getManagedEmbeddablesStore } from './generic_embeddable_store'; +import { + ReduxContainerContextServices, + ReduxEmbeddableContextServices, + ReduxEmbeddableWrapperProps, +} from './types'; +import { ReduxEmbeddableContext, useReduxEmbeddableContext } from './redux_embeddable_context'; + +const getDefaultProps = (): Required< + Pick, 'diffInput'> +> => ({ + diffInput: (a, b) => { + const differences: Partial = {}; + const allKeys = [...Object.keys(a), ...Object.keys(b)] as Array; + allKeys.forEach((key) => { + if (!isEqual(a[key], b[key])) differences[key] = a[key]; + }); + return differences; + }, +}); + +const embeddableIsContainer = ( + embeddable: IEmbeddable +): embeddable is IContainer => embeddable.isContainer; + +/** + * Place this wrapper around the react component when rendering an embeddable to automatically set up + * redux for use with the embeddable via the supplied reducers. Any child components can then use ReduxEmbeddableContext + * or ReduxContainerContext to interface with the state of the embeddable. + */ +export const ReduxEmbeddableWrapper = ( + props: PropsWithChildren> +) => { + const { embeddable, reducers, diffInput } = useMemo( + () => ({ ...getDefaultProps(), ...props }), + [props] + ); + + const containerActions: ReduxContainerContextServices['containerActions'] | undefined = + useMemo(() => { + if (embeddableIsContainer(embeddable)) { + return { + untilEmbeddableLoaded: embeddable.untilEmbeddableLoaded.bind(embeddable), + updateInputForChild: embeddable.updateInputForChild.bind(embeddable), + removeEmbeddable: embeddable.removeEmbeddable.bind(embeddable), + addNewEmbeddable: embeddable.addNewEmbeddable.bind(embeddable), + }; + } + return; + }, [embeddable]); + + const reduxEmbeddableContext: ReduxEmbeddableContextServices | ReduxContainerContextServices = + useMemo(() => { + const key = `${embeddable.type}_${embeddable.id}`; + + // A generic reducer used to update redux state when the embeddable input changes + const updateEmbeddableReduxState = ( + state: Draft, + action: PayloadAction> + ) => { + return { ...state, ...action.payload }; + }; + + const slice = createSlice>({ + initialState: embeddable.getInput(), + name: key, + reducers: { ...reducers, updateEmbeddableReduxState }, + }); + const store = getManagedEmbeddablesStore(); + + store.injectReducer({ + key, + asyncReducer: slice.reducer, + }); + + const useEmbeddableSelector: TypedUseSelectorHook = () => + useSelector((state: ReturnType) => state[key]); + + return { + useEmbeddableDispatch: () => useDispatch(), + useEmbeddableSelector, + actions: slice.actions as ReduxEmbeddableContextServices['actions'], + containerActions, + }; + }, [reducers, embeddable, containerActions]); + + return ( + + + + {props.children} + + + + ); +}; + +interface ReduxEmbeddableSyncProps { + diffInput: (a: InputType, b: InputType) => Partial; + embeddable: IEmbeddable; +} + +/** + * This component uses the context from the embeddable wrapper to set up a generic two-way binding between the embeddable input and + * the redux store. a custom diffInput function can be provided, this function should always prioritize input A over input B. + */ +const ReduxEmbeddableSync = ({ + embeddable, + diffInput, + children, +}: PropsWithChildren>) => { + const { + useEmbeddableSelector, + useEmbeddableDispatch, + actions: { updateEmbeddableReduxState }, + } = useReduxEmbeddableContext(); + + const dispatch = useEmbeddableDispatch(); + const currentState = useEmbeddableSelector((state) => state); + const stateRef = useRef(currentState); + + // When Embeddable Input changes, push differences to redux. + useEffect(() => { + embeddable.getInput$().subscribe(() => { + const differences = diffInput(embeddable.getInput(), stateRef.current); + if (differences && Object.keys(differences).length > 0) { + dispatch(updateEmbeddableReduxState(differences)); + } + }); + }, [diffInput, dispatch, embeddable, updateEmbeddableReduxState]); + + // When redux state changes, push differences to Embeddable Input. + useEffect(() => { + stateRef.current = currentState; + const differences = diffInput(currentState, embeddable.getInput()); + if (differences && Object.keys(differences).length > 0) { + embeddable.updateInput(differences); + } + }, [currentState, diffInput, embeddable]); + + return <>{children}; +}; diff --git a/src/plugins/presentation_util/public/components/redux_embeddables/types.ts b/src/plugins/presentation_util/public/components/redux_embeddables/types.ts new file mode 100644 index 0000000000000..118b5d340528e --- /dev/null +++ b/src/plugins/presentation_util/public/components/redux_embeddables/types.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + ActionCreatorWithPayload, + AnyAction, + CaseReducer, + Dispatch, + PayloadAction, +} from '@reduxjs/toolkit'; +import { TypedUseSelectorHook } from 'react-redux'; +import { + EmbeddableInput, + EmbeddableOutput, + IContainer, + IEmbeddable, +} from '../../../../embeddable/public'; + +export interface GenericEmbeddableReducers { + /** + * PayloadAction of type any is strategic here because we want to allow payloads of any shape in generic reducers. + * This type will be overridden to remove any and be type safe when returned by ReduxEmbeddableContextServices. + */ + [key: string]: CaseReducer>; +} + +export interface ReduxEmbeddableWrapperProps { + embeddable: IEmbeddable; + reducers: GenericEmbeddableReducers; + diffInput?: (a: InputType, b: InputType) => Partial; +} + +/** + * This context allows components underneath the redux embeddable wrapper to get access to the actions, selector, dispatch, and containerActions. + */ +export interface ReduxEmbeddableContextServices< + InputType extends EmbeddableInput = EmbeddableInput, + ReducerType extends GenericEmbeddableReducers = GenericEmbeddableReducers +> { + actions: { + [Property in keyof ReducerType]: ActionCreatorWithPayload< + Parameters[1]['payload'] + >; + } & { updateEmbeddableReduxState: ActionCreatorWithPayload> }; + useEmbeddableSelector: TypedUseSelectorHook; + useEmbeddableDispatch: () => Dispatch; +} + +export type ReduxContainerContextServices< + InputType extends EmbeddableInput = EmbeddableInput, + ReducerType extends GenericEmbeddableReducers = GenericEmbeddableReducers +> = ReduxEmbeddableContextServices & { + containerActions: Pick< + IContainer, + 'untilEmbeddableLoaded' | 'removeEmbeddable' | 'addNewEmbeddable' | 'updateInputForChild' + >; +}; diff --git a/src/plugins/presentation_util/public/mocks.ts b/src/plugins/presentation_util/public/mocks.ts index 91c461646c280..ddb02ce464e22 100644 --- a/src/plugins/presentation_util/public/mocks.ts +++ b/src/plugins/presentation_util/public/mocks.ts @@ -17,6 +17,7 @@ const createStartContract = (coreStart: CoreStart): PresentationUtilPluginStart const startContract: PresentationUtilPluginStart = { ContextProvider: pluginServices.getContextProvider(), labsService: pluginServices.getServices().labs, + controlsService: pluginServices.getServices().controls, }; return startContract; }; diff --git a/src/plugins/presentation_util/public/plugin.ts b/src/plugins/presentation_util/public/plugin.ts index f34bd2f1f8afe..f697f1a29eb82 100644 --- a/src/plugins/presentation_util/public/plugin.ts +++ b/src/plugins/presentation_util/public/plugin.ts @@ -39,6 +39,7 @@ export class PresentationUtilPlugin pluginServices.setRegistry(registry.start({ coreStart, startPlugins })); return { ContextProvider: pluginServices.getContextProvider(), + controlsService: pluginServices.getServices().controls, labsService: pluginServices.getServices().labs, }; } diff --git a/src/plugins/presentation_util/public/services/controls.ts b/src/plugins/presentation_util/public/services/controls.ts new file mode 100644 index 0000000000000..197e986381b10 --- /dev/null +++ b/src/plugins/presentation_util/public/services/controls.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Filter } from '@kbn/es-query'; +import { Query, TimeRange } from '../../../data/public'; +import { + EmbeddableFactory, + EmbeddableInput, + EmbeddableOutput, + IEmbeddable, +} from '../../../embeddable/public'; + +/** + * Control embeddable types + */ +export type InputControlFactory = EmbeddableFactory< + InputControlInput, + InputControlOutput, + InputControlEmbeddable +>; + +export type InputControlInput = EmbeddableInput & { + query?: Query; + filters?: Filter[]; + timeRange?: TimeRange; + twoLineLayout?: boolean; +}; + +export type InputControlOutput = EmbeddableOutput & { + filters?: Filter[]; +}; + +export type InputControlEmbeddable< + TInputControlEmbeddableInput extends InputControlInput = InputControlInput, + TInputControlEmbeddableOutput extends InputControlOutput = InputControlOutput +> = IEmbeddable; + +export interface ControlTypeRegistry { + [key: string]: InputControlFactory; +} + +export interface PresentationControlsService { + registerInputControlType: (factory: InputControlFactory) => void; + + getControlFactory: < + I extends InputControlInput = InputControlInput, + O extends InputControlOutput = InputControlOutput, + E extends InputControlEmbeddable = InputControlEmbeddable + >( + type: string + ) => EmbeddableFactory; + + getInputControlTypes: () => string[]; +} + +export const getCommonControlsService = () => { + const controlsFactoriesMap: ControlTypeRegistry = {}; + + const registerInputControlType = (factory: InputControlFactory) => { + controlsFactoriesMap[factory.type] = factory; + }; + + const getControlFactory = < + I extends InputControlInput = InputControlInput, + O extends InputControlOutput = InputControlOutput, + E extends InputControlEmbeddable = InputControlEmbeddable + >( + type: string + ) => { + return controlsFactoriesMap[type] as EmbeddableFactory; + }; + + const getInputControlTypes = () => Object.keys(controlsFactoriesMap); + + return { + registerInputControlType, + getControlFactory, + getInputControlTypes, + }; +}; diff --git a/src/plugins/presentation_util/public/services/index.ts b/src/plugins/presentation_util/public/services/index.ts index c622ad82bb888..21012971ca86d 100644 --- a/src/plugins/presentation_util/public/services/index.ts +++ b/src/plugins/presentation_util/public/services/index.ts @@ -13,6 +13,7 @@ import { PresentationDashboardsService } from './dashboards'; import { PresentationLabsService } from './labs'; import { registry as stubRegistry } from './stub'; import { PresentationOverlaysService } from './overlays'; +import { PresentationControlsService } from './controls'; export { PresentationCapabilitiesService } from './capabilities'; export { PresentationDashboardsService } from './dashboards'; @@ -21,6 +22,7 @@ export interface PresentationUtilServices { dashboards: PresentationDashboardsService; capabilities: PresentationCapabilitiesService; overlays: PresentationOverlaysService; + controls: PresentationControlsService; labs: PresentationLabsService; } @@ -31,5 +33,6 @@ export const getStubPluginServices = (): PresentationUtilPluginStart => { return { ContextProvider: pluginServices.getContextProvider(), labsService: pluginServices.getServices().labs, + controlsService: pluginServices.getServices().controls, }; }; diff --git a/src/plugins/presentation_util/public/components/controls/index.ts b/src/plugins/presentation_util/public/services/kibana/controls.ts similarity index 54% rename from src/plugins/presentation_util/public/components/controls/index.ts rename to src/plugins/presentation_util/public/services/kibana/controls.ts index 5c2d5b68ae2e0..e5dc84a3dd645 100644 --- a/src/plugins/presentation_util/public/components/controls/index.ts +++ b/src/plugins/presentation_util/public/services/kibana/controls.ts @@ -5,3 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ + +import { PluginServiceFactory } from '../create'; +import { getCommonControlsService, PresentationControlsService } from '../controls'; + +export type ControlsServiceFactory = PluginServiceFactory; +export const controlsServiceFactory = () => getCommonControlsService(); diff --git a/src/plugins/presentation_util/public/services/kibana/index.ts b/src/plugins/presentation_util/public/services/kibana/index.ts index 8a9a28606f24b..48c921bff1efd 100644 --- a/src/plugins/presentation_util/public/services/kibana/index.ts +++ b/src/plugins/presentation_util/public/services/kibana/index.ts @@ -18,6 +18,7 @@ import { } from '../create'; import { PresentationUtilPluginStartDeps } from '../../types'; import { PresentationUtilServices } from '..'; +import { controlsServiceFactory } from './controls'; export { capabilitiesServiceFactory } from './capabilities'; export { dashboardsServiceFactory } from './dashboards'; @@ -32,6 +33,7 @@ export const providers: PluginServiceProviders< labs: new PluginServiceProvider(labsServiceFactory), dashboards: new PluginServiceProvider(dashboardsServiceFactory), overlays: new PluginServiceProvider(overlaysServiceFactory), + controls: new PluginServiceProvider(controlsServiceFactory), }; export const registry = new PluginServiceRegistry< diff --git a/src/plugins/presentation_util/public/services/storybook/controls.ts b/src/plugins/presentation_util/public/services/storybook/controls.ts new file mode 100644 index 0000000000000..e5dc84a3dd645 --- /dev/null +++ b/src/plugins/presentation_util/public/services/storybook/controls.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PluginServiceFactory } from '../create'; +import { getCommonControlsService, PresentationControlsService } from '../controls'; + +export type ControlsServiceFactory = PluginServiceFactory; +export const controlsServiceFactory = () => getCommonControlsService(); diff --git a/src/plugins/presentation_util/public/services/storybook/index.ts b/src/plugins/presentation_util/public/services/storybook/index.ts index 1ce1eb72848c9..9de4934d51300 100644 --- a/src/plugins/presentation_util/public/services/storybook/index.ts +++ b/src/plugins/presentation_util/public/services/storybook/index.ts @@ -6,12 +6,18 @@ * Side Public License, v 1. */ -import { PluginServices, PluginServiceProviders, PluginServiceProvider } from '../create'; +import { + PluginServices, + PluginServiceProviders, + PluginServiceProvider, + PluginServiceRegistry, +} from '../create'; import { dashboardsServiceFactory } from '../stub/dashboards'; import { labsServiceFactory } from './labs'; import { capabilitiesServiceFactory } from './capabilities'; import { PresentationUtilServices } from '..'; import { overlaysServiceFactory } from './overlays'; +import { controlsServiceFactory } from './controls'; export { PluginServiceProviders, PluginServiceProvider, PluginServiceRegistry } from '../create'; export { PresentationUtilServices } from '..'; @@ -27,7 +33,10 @@ export const providers: PluginServiceProviders(); + +export const registry = new PluginServiceRegistry(providers); diff --git a/src/plugins/presentation_util/public/services/stub/controls.ts b/src/plugins/presentation_util/public/services/stub/controls.ts new file mode 100644 index 0000000000000..e5dc84a3dd645 --- /dev/null +++ b/src/plugins/presentation_util/public/services/stub/controls.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PluginServiceFactory } from '../create'; +import { getCommonControlsService, PresentationControlsService } from '../controls'; + +export type ControlsServiceFactory = PluginServiceFactory; +export const controlsServiceFactory = () => getCommonControlsService(); diff --git a/src/plugins/presentation_util/public/services/stub/index.ts b/src/plugins/presentation_util/public/services/stub/index.ts index 61dca47427531..35aabdb465b14 100644 --- a/src/plugins/presentation_util/public/services/stub/index.ts +++ b/src/plugins/presentation_util/public/services/stub/index.ts @@ -12,7 +12,7 @@ import { labsServiceFactory } from './labs'; import { PluginServiceProviders, PluginServiceProvider, PluginServiceRegistry } from '../create'; import { PresentationUtilServices } from '..'; import { overlaysServiceFactory } from './overlays'; - +import { controlsServiceFactory } from './controls'; export { dashboardsServiceFactory } from './dashboards'; export { capabilitiesServiceFactory } from './capabilities'; @@ -20,6 +20,7 @@ export const providers: PluginServiceProviders = { dashboards: new PluginServiceProvider(dashboardsServiceFactory), capabilities: new PluginServiceProvider(capabilitiesServiceFactory), overlays: new PluginServiceProvider(overlaysServiceFactory), + controls: new PluginServiceProvider(controlsServiceFactory), labs: new PluginServiceProvider(labsServiceFactory), }; diff --git a/src/plugins/presentation_util/public/types.ts b/src/plugins/presentation_util/public/types.ts index 05779ffb206c4..3903d1bc2786e 100644 --- a/src/plugins/presentation_util/public/types.ts +++ b/src/plugins/presentation_util/public/types.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { PresentationControlsService } from './services/controls'; import { PresentationLabsService } from './services/labs'; // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -14,6 +15,7 @@ export interface PresentationUtilPluginSetup {} export interface PresentationUtilPluginStart { ContextProvider: React.FC; labsService: PresentationLabsService; + controlsService: PresentationControlsService; } // eslint-disable-next-line @typescript-eslint/no-empty-interface From 461f9f65ccb5e3e9444d2ce69013fd9856f49b30 Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Wed, 13 Oct 2021 21:12:21 -0400 Subject: [PATCH 30/35] fix package.json: (#114936) --- package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/package.json b/package.json index 6e4a37863bc82..f526f357ff347 100644 --- a/package.json +++ b/package.json @@ -91,9 +91,6 @@ "yarn": "^1.21.1" }, "dependencies": { - "@dnd-kit/core": "^3.1.1", - "@dnd-kit/sortable": "^4.0.0", - "@dnd-kit/utilities": "^2.0.0", "@babel/runtime": "^7.15.4", "@dnd-kit/core": "^3.1.1", "@dnd-kit/sortable": "^4.0.0", From 5647de3b4a32815a5c8c4df1a35072bb3d1db7db Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Wed, 13 Oct 2021 18:18:06 -0700 Subject: [PATCH 31/35] [ci] Fixes Bazel cache writes (#114915) Signed-off-by: Tyler Smalley --- .../scripts/common/persist_bazel_cache.sh | 14 +++++++++++ .buildkite/scripts/common/setup_bazel.sh | 24 ------------------- .../steps/on_merge_build_and_metrics.sh | 2 +- src/dev/ci_setup/.bazelrc-ci | 13 +++++----- src/dev/ci_setup/.bazelrc-ci.common | 11 --------- 5 files changed, 21 insertions(+), 43 deletions(-) create mode 100755 .buildkite/scripts/common/persist_bazel_cache.sh delete mode 100755 .buildkite/scripts/common/setup_bazel.sh delete mode 100644 src/dev/ci_setup/.bazelrc-ci.common diff --git a/.buildkite/scripts/common/persist_bazel_cache.sh b/.buildkite/scripts/common/persist_bazel_cache.sh new file mode 100755 index 0000000000000..597ab0947c267 --- /dev/null +++ b/.buildkite/scripts/common/persist_bazel_cache.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common/util.sh + +KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key) +export KIBANA_BUILDBUDDY_CI_API_KEY + +cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$KIBANA_DIR/.bazelrc" + +### +### append auth token to buildbuddy into "$HOME/.bazelrc"; +### +echo "# Appended by .buildkite/scripts/persist_bazel_cache.sh" >> "$KIBANA_DIR/.bazelrc" +echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$KIBANA_DIR/.bazelrc" diff --git a/.buildkite/scripts/common/setup_bazel.sh b/.buildkite/scripts/common/setup_bazel.sh deleted file mode 100755 index bbd1c58497172..0000000000000 --- a/.buildkite/scripts/common/setup_bazel.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common/util.sh - -KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key) -export KIBANA_BUILDBUDDY_CI_API_KEY - -cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc" - -### -### append auth token to buildbuddy into "$HOME/.bazelrc"; -### -echo "# Appended by .buildkite/scripts/setup_bazel.sh" >> "$HOME/.bazelrc" -echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc" - -### -### remove write permissions on buildbuddy remote cache for prs -### -if [[ "${BUILDKITE_PULL_REQUEST:-}" && "$BUILDKITE_PULL_REQUEST" != "false" ]] ; then - { - echo "# Uploads logs & artifacts without writing to cache" - echo "build --noremote_upload_local_results" - } >> "$HOME/.bazelrc" -fi diff --git a/.buildkite/scripts/steps/on_merge_build_and_metrics.sh b/.buildkite/scripts/steps/on_merge_build_and_metrics.sh index b24e585e70735..315ba08f8719b 100755 --- a/.buildkite/scripts/steps/on_merge_build_and_metrics.sh +++ b/.buildkite/scripts/steps/on_merge_build_and_metrics.sh @@ -3,7 +3,7 @@ set -euo pipefail # Write Bazel cache for Linux -.buildkite/scripts/common/setup_bazel.sh +.buildkite/scripts/common/persist_bazel_cache.sh .buildkite/scripts/bootstrap.sh .buildkite/scripts/build_kibana.sh diff --git a/src/dev/ci_setup/.bazelrc-ci b/src/dev/ci_setup/.bazelrc-ci index ef6fab3a30590..9aee657f37bcb 100644 --- a/src/dev/ci_setup/.bazelrc-ci +++ b/src/dev/ci_setup/.bazelrc-ci @@ -1,16 +1,15 @@ -# Inspired on https://github.com/angular/angular/blob/master/.circleci/bazel.linux.rc -# These options are only enabled when running on CI -# That is done by copying this file into "$HOME/.bazelrc" which loads after the .bazelrc into the workspace +# Used in the on-merge job to persist the Bazel cache to BuildBuddy +# from: .buildkite/scripts/common/persist_bazel_cache.sh -# Import and load bazelrc common settings for ci env -try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.common +import %workspace%/.bazelrc.common # BuildBuddy settings -## Remote settings including cache build --bes_results_url=https://app.buildbuddy.io/invocation/ build --bes_backend=grpcs://cloud.buildbuddy.io build --remote_cache=grpcs://cloud.buildbuddy.io build --remote_timeout=3600 +# --remote_header=x-buildbuddy-api-key= # appended in CI script -## Metadata settings +# Metadata settings build --build_metadata=ROLE=CI +build --workspace_status_command="node ./src/dev/bazel_workspace_status.js" diff --git a/src/dev/ci_setup/.bazelrc-ci.common b/src/dev/ci_setup/.bazelrc-ci.common deleted file mode 100644 index 56a5ee8d30cd6..0000000000000 --- a/src/dev/ci_setup/.bazelrc-ci.common +++ /dev/null @@ -1,11 +0,0 @@ -# Inspired on https://github.com/angular/angular/blob/master/.circleci/bazel.common.rc -# Settings in this file should be OS agnostic - -# Don't be spammy in the logs -build --noshow_progress - -# More details on failures -build --verbose_failures=true - -## Avoid to keep connections to build event backend connections alive across builds -build --keep_backend_build_event_connections_alive=false From 423b0e801fb29dcf02f6588fa8f888b9aebe2f15 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 14 Oct 2021 03:35:17 +0100 Subject: [PATCH 32/35] chore(NA): fixes a typo on persist_bazel_cache.sh comment (#114943) --- .buildkite/scripts/common/persist_bazel_cache.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/common/persist_bazel_cache.sh b/.buildkite/scripts/common/persist_bazel_cache.sh index 597ab0947c267..357805c11acec 100755 --- a/.buildkite/scripts/common/persist_bazel_cache.sh +++ b/.buildkite/scripts/common/persist_bazel_cache.sh @@ -5,10 +5,11 @@ source .buildkite/scripts/common/util.sh KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key) export KIBANA_BUILDBUDDY_CI_API_KEY +# overwrites the file checkout .bazelrc file with the one intended for CI env cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$KIBANA_DIR/.bazelrc" ### -### append auth token to buildbuddy into "$HOME/.bazelrc"; +### append auth token to buildbuddy into "$KIBANA_DIR/.bazelrc"; ### echo "# Appended by .buildkite/scripts/persist_bazel_cache.sh" >> "$KIBANA_DIR/.bazelrc" echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$KIBANA_DIR/.bazelrc" From 86f0733e5642a10b77025f36199d895282c1e601 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 13 Oct 2021 19:49:16 -0700 Subject: [PATCH 33/35] Clean up inaccurate comments (#114935) --- .../create_package_policy_page/step_configure_package.tsx | 1 - x-pack/plugins/fleet/server/services/epm/packages/get.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx index 1ff5d20baec06..390e540f1b10f 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx @@ -49,7 +49,6 @@ export const StepConfigurePackagePolicy: React.FunctionComponent<{ ); // Configure inputs (and their streams) - // Assume packages only export one config template for now const renderConfigureInputs = () => packagePolicyTemplates.length ? ( <> diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index cf847cdf62bc2..8d3c1fbe0daa4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -69,7 +69,6 @@ export async function getPackages( } // Get package names for packages which cannot have more than one package policy on an agent policy -// Assume packages only export one policy template for now export async function getLimitedPackages(options: { savedObjectsClient: SavedObjectsClientContract; }): Promise { From 69a6cf329ce8be83b48a102e28983b7d0ca11ab8 Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Wed, 13 Oct 2021 20:32:43 -0700 Subject: [PATCH 34/35] Fixing exceptions export format (#114920) ### Summary Fixing exceptions export format and adding integration tests for it. --- .../src/api/index.ts | 2 +- .../kbn-securitysolution-utils/src/index.ts | 1 + .../transform_data_to_ndjson/index.test.ts | 88 ++++++++++ .../src/transform_data_to_ndjson/index.ts | 16 ++ .../lists/public/exceptions/api.test.ts | 2 +- .../routes/export_exception_list_route.ts | 42 +++-- .../downloads/test_exception_list.ndjson | 2 + .../exceptions/exceptions_table.spec.ts | 2 +- .../cypress/objects/exception.ts | 3 +- .../detection_engine/rules/get_export_all.ts | 3 +- .../rules/get_export_by_object_ids.ts | 2 +- .../server/lib/telemetry/sender.ts | 3 +- .../timelines/export_timelines/helpers.ts | 3 +- .../create_stream_from_ndjson.test.ts | 71 -------- .../read_stream/create_stream_from_ndjson.ts | 9 - .../tests/export_exception_list.ts | 155 ++++++++++++++++++ .../security_and_spaces/tests/index.ts | 1 + 17 files changed, 292 insertions(+), 113 deletions(-) create mode 100644 packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.test.ts create mode 100644 packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.ts create mode 100644 x-pack/plugins/security_solution/cypress/downloads/test_exception_list.ndjson delete mode 100644 x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.test.ts create mode 100644 x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts diff --git a/packages/kbn-securitysolution-list-api/src/api/index.ts b/packages/kbn-securitysolution-list-api/src/api/index.ts index d70417a29971f..77c50fb32c299 100644 --- a/packages/kbn-securitysolution-list-api/src/api/index.ts +++ b/packages/kbn-securitysolution-list-api/src/api/index.ts @@ -558,7 +558,7 @@ export const exportExceptionList = async ({ signal, }: ExportExceptionListProps): Promise => http.fetch(`${EXCEPTION_LIST_URL}/_export`, { - method: 'GET', + method: 'POST', query: { id, list_id: listId, namespace_type: namespaceType }, signal, }); diff --git a/packages/kbn-securitysolution-utils/src/index.ts b/packages/kbn-securitysolution-utils/src/index.ts index 0bb36c590ffdf..755bbd2203dff 100644 --- a/packages/kbn-securitysolution-utils/src/index.ts +++ b/packages/kbn-securitysolution-utils/src/index.ts @@ -7,3 +7,4 @@ */ export * from './add_remove_id_to_item'; +export * from './transform_data_to_ndjson'; diff --git a/packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.test.ts b/packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.test.ts new file mode 100644 index 0000000000000..b10626357f5b1 --- /dev/null +++ b/packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { transformDataToNdjson } from './'; + +export const ANCHOR_DATE = '2020-02-20T03:57:54.037Z'; + +const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE) => ({ + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(anchorDate).toISOString(), + updated_at: new Date(anchorDate).toISOString(), + created_by: 'elastic', + description: 'some description', + enabled: true, + false_positives: ['false positive 1', 'false positive 2'], + from: 'now-6m', + immutable: false, + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + references: ['test 1', 'test 2'], + severity: 'high', + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: [], +}); + +describe('transformDataToNdjson', () => { + test('if rules are empty it returns an empty string', () => { + const ruleNdjson = transformDataToNdjson([]); + expect(ruleNdjson).toEqual(''); + }); + + test('single rule will transform with new line ending character for ndjson', () => { + const rule = getRulesSchemaMock(); + const ruleNdjson = transformDataToNdjson([rule]); + expect(ruleNdjson.endsWith('\n')).toBe(true); + }); + + test('multiple rules will transform with two new line ending characters for ndjson', () => { + const result1 = getRulesSchemaMock(); + const result2 = getRulesSchemaMock(); + result2.id = 'some other id'; + result2.rule_id = 'some other id'; + result2.name = 'Some other rule'; + + const ruleNdjson = transformDataToNdjson([result1, result2]); + // this is how we count characters in JavaScript :-) + const count = ruleNdjson.split('\n').length - 1; + expect(count).toBe(2); + }); + + test('you can parse two rules back out without errors', () => { + const result1 = getRulesSchemaMock(); + const result2 = getRulesSchemaMock(); + result2.id = 'some other id'; + result2.rule_id = 'some other id'; + result2.name = 'Some other rule'; + + const ruleNdjson = transformDataToNdjson([result1, result2]); + const ruleStrings = ruleNdjson.split('\n'); + const reParsed1 = JSON.parse(ruleStrings[0]); + const reParsed2 = JSON.parse(ruleStrings[1]); + expect(reParsed1).toEqual(result1); + expect(reParsed2).toEqual(result2); + }); +}); diff --git a/packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.ts b/packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.ts new file mode 100644 index 0000000000000..66a500731f497 --- /dev/null +++ b/packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const transformDataToNdjson = (data: unknown[]): string => { + if (data.length !== 0) { + const dataString = data.map((item) => JSON.stringify(item)).join('\n'); + return `${dataString}\n`; + } else { + return ''; + } +}; diff --git a/x-pack/plugins/lists/public/exceptions/api.test.ts b/x-pack/plugins/lists/public/exceptions/api.test.ts index a196999d14943..65c11bfc1dfd0 100644 --- a/x-pack/plugins/lists/public/exceptions/api.test.ts +++ b/x-pack/plugins/lists/public/exceptions/api.test.ts @@ -759,7 +759,7 @@ describe('Exceptions Lists API', () => { }); expect(httpMock.fetch).toHaveBeenCalledWith('/api/exception_lists/_export', { - method: 'GET', + method: 'POST', query: { id: 'some-id', list_id: 'list-id', diff --git a/x-pack/plugins/lists/server/routes/export_exception_list_route.ts b/x-pack/plugins/lists/server/routes/export_exception_list_route.ts index a238d0e6529ff..aa30c8a7d435d 100644 --- a/x-pack/plugins/lists/server/routes/export_exception_list_route.ts +++ b/x-pack/plugins/lists/server/routes/export_exception_list_route.ts @@ -6,6 +6,7 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; +import { transformDataToNdjson } from '@kbn/securitysolution-utils'; import { exportExceptionListQuerySchema } from '@kbn/securitysolution-io-ts-list-types'; import { EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants'; @@ -14,7 +15,7 @@ import type { ListsPluginRouter } from '../types'; import { buildRouteValidation, buildSiemResponse, getExceptionListClient } from './utils'; export const exportExceptionListRoute = (router: ListsPluginRouter): void => { - router.get( + router.post( { options: { tags: ['access:lists-read'], @@ -26,6 +27,7 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => { }, async (context, request, response) => { const siemResponse = buildSiemResponse(response); + try { const { id, list_id: listId, namespace_type: namespaceType } = request.query; const exceptionLists = getExceptionListClient(context); @@ -37,11 +39,10 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => { if (exceptionList == null) { return siemResponse.error({ - body: `list_id: ${listId} does not exist`, + body: `exception list with list_id: ${listId} does not exist`, statusCode: 400, }); } else { - const { exportData: exportList } = getExport([exceptionList]); const listItems = await exceptionLists.findExceptionListItem({ filter: undefined, listId, @@ -51,19 +52,15 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => { sortField: 'exception-list.created_at', sortOrder: 'desc', }); + const exceptionItems = listItems?.data ?? []; - const { exportData: exportListItems, exportDetails } = getExport(listItems?.data ?? []); - - const responseBody = [ - exportList, - exportListItems, - { exception_list_items_details: exportDetails }, - ]; + const { exportData } = getExport([exceptionList, ...exceptionItems]); + const { exportDetails } = getExportDetails(exceptionItems); // TODO: Allow the API to override the name of the file to export const fileName = exceptionList.list_id; return response.ok({ - body: transformDataToNdjson(responseBody), + body: `${exportData}${exportDetails}`, headers: { 'Content-Disposition': `attachment; filename="${fileName}"`, 'Content-Type': 'application/ndjson', @@ -81,24 +78,23 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => { ); }; -const transformDataToNdjson = (data: unknown[]): string => { - if (data.length !== 0) { - const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n'); - return `${dataString}\n`; - } else { - return ''; - } -}; - export const getExport = ( data: unknown[] ): { exportData: string; - exportDetails: string; } => { const ndjson = transformDataToNdjson(data); + + return { exportData: ndjson }; +}; + +export const getExportDetails = ( + items: unknown[] +): { + exportDetails: string; +} => { const exportDetails = JSON.stringify({ - exported_count: data.length, + exported_list_items_count: items.length, }); - return { exportData: ndjson, exportDetails: `${exportDetails}\n` }; + return { exportDetails: `${exportDetails}\n` }; }; diff --git a/x-pack/plugins/security_solution/cypress/downloads/test_exception_list.ndjson b/x-pack/plugins/security_solution/cypress/downloads/test_exception_list.ndjson new file mode 100644 index 0000000000000..54420eff29e0d --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/downloads/test_exception_list.ndjson @@ -0,0 +1,2 @@ +{"_version":"WzQyNjA0LDFd","created_at":"2021-10-14T01:30:22.034Z","created_by":"elastic","description":"Test exception list description","id":"4c65a230-2c8e-11ec-be1c-2bbdec602f88","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"b04983b4-1617-441c-bb6c-c729281fa2e9","type":"detection","updated_at":"2021-10-14T01:30:22.036Z","updated_by":"elastic","version":1} +{"exported_list_items_count":0} diff --git a/x-pack/plugins/security_solution/cypress/integration/exceptions/exceptions_table.spec.ts b/x-pack/plugins/security_solution/cypress/integration/exceptions/exceptions_table.spec.ts index 8530f949664b8..c8b6f73912acf 100644 --- a/x-pack/plugins/security_solution/cypress/integration/exceptions/exceptions_table.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/exceptions/exceptions_table.spec.ts @@ -81,7 +81,7 @@ describe('Exceptions Table', () => { cy.wait('@export').then(({ response }) => cy - .wrap(response?.body!) + .wrap(response?.body) .should('eql', expectedExportedExceptionList(this.exceptionListResponse)) ); }); diff --git a/x-pack/plugins/security_solution/cypress/objects/exception.ts b/x-pack/plugins/security_solution/cypress/objects/exception.ts index 81c3b885ab94d..b772924697148 100644 --- a/x-pack/plugins/security_solution/cypress/objects/exception.ts +++ b/x-pack/plugins/security_solution/cypress/objects/exception.ts @@ -41,6 +41,5 @@ export const expectedExportedExceptionList = ( exceptionListResponse: Cypress.Response ): string => { const jsonrule = exceptionListResponse.body; - - return `"{\\"_version\\":\\"${jsonrule._version}\\",\\"created_at\\":\\"${jsonrule.created_at}\\",\\"created_by\\":\\"elastic\\",\\"description\\":\\"${jsonrule.description}\\",\\"id\\":\\"${jsonrule.id}\\",\\"immutable\\":false,\\"list_id\\":\\"test_exception_list\\",\\"name\\":\\"Test exception list\\",\\"namespace_type\\":\\"single\\",\\"os_types\\":[],\\"tags\\":[],\\"tie_breaker_id\\":\\"${jsonrule.tie_breaker_id}\\",\\"type\\":\\"detection\\",\\"updated_at\\":\\"${jsonrule.updated_at}\\",\\"updated_by\\":\\"elastic\\",\\"version\\":1}\\n"\n""\n{"exception_list_items_details":"{\\"exported_count\\":0}\\n"}\n`; + return `{"_version":"${jsonrule._version}","created_at":"${jsonrule.created_at}","created_by":"elastic","description":"${jsonrule.description}","id":"${jsonrule.id}","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"${jsonrule.tie_breaker_id}","type":"detection","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","version":1}\n{"exported_list_items_count":0}\n`; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.ts index f44471e6e26f9..71079ccefc97a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.ts @@ -5,11 +5,12 @@ * 2.0. */ +import { transformDataToNdjson } from '@kbn/securitysolution-utils'; + import { RulesClient } from '../../../../../alerting/server'; import { getNonPackagedRules } from './get_existing_prepackaged_rules'; import { getExportDetailsNdjson } from './get_export_details_ndjson'; import { transformAlertsToRules } from '../routes/rules/utils'; -import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson'; export const getExportAll = async ( rulesClient: RulesClient, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.ts index 31a7604306de7..4cf3ad9133a71 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.ts @@ -6,13 +6,13 @@ */ import { chunk } from 'lodash'; +import { transformDataToNdjson } from '@kbn/securitysolution-utils'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { RulesClient } from '../../../../../alerting/server'; import { getExportDetailsNdjson } from './get_export_details_ndjson'; import { isAlertType } from '../rules/types'; import { transformAlertToRule } from '../routes/rules/utils'; -import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson'; import { INTERNAL_RULE_ID_KEY } from '../../../../common/constants'; import { findRules } from './find_rules'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 2966fa3decc26..7c9906d0eae48 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -8,10 +8,11 @@ import { cloneDeep } from 'lodash'; import axios from 'axios'; import { URL } from 'url'; +import { transformDataToNdjson } from '@kbn/securitysolution-utils'; + import { Logger } from 'src/core/server'; import { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; import { UsageCounter } from 'src/plugins/usage_collection/server'; -import { transformDataToNdjson } from '../../utils/read_stream/create_stream_from_ndjson'; import { TaskManagerSetupContract, TaskManagerStartContract, diff --git a/x-pack/plugins/security_solution/server/lib/timeline/routes/timelines/export_timelines/helpers.ts b/x-pack/plugins/security_solution/server/lib/timeline/routes/timelines/export_timelines/helpers.ts index a33b8be0c2f31..c857e7fa38a27 100644 --- a/x-pack/plugins/security_solution/server/lib/timeline/routes/timelines/export_timelines/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/timeline/routes/timelines/export_timelines/helpers.ts @@ -6,6 +6,7 @@ */ import { omit } from 'lodash/fp'; +import { transformDataToNdjson } from '@kbn/securitysolution-utils'; import { ExportedTimelines, @@ -15,8 +16,6 @@ import { import { NoteSavedObject } from '../../../../../../common/types/timeline/note'; import { PinnedEventSavedObject } from '../../../../../../common/types/timeline/pinned_event'; -import { transformDataToNdjson } from '../../../../../utils/read_stream/create_stream_from_ndjson'; - import { FrameworkRequest } from '../../../../framework'; import * as noteLib from '../../../saved_object/notes'; import * as pinnedEventLib from '../../../saved_object/pinned_events'; diff --git a/x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.test.ts b/x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.test.ts deleted file mode 100644 index c3163da6ac949..0000000000000 --- a/x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { transformDataToNdjson } from './create_stream_from_ndjson'; -import { ImportRulesSchemaDecoded } from '../../../common/detection_engine/schemas/request/import_rules_schema'; -import { getRulesSchemaMock } from '../../../common/detection_engine/schemas/response/rules_schema.mocks'; - -export const getOutputSample = (): Partial => ({ - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', -}); - -export const getSampleAsNdjson = (sample: Partial): string => { - return `${JSON.stringify(sample)}\n`; -}; - -describe('create_rules_stream_from_ndjson', () => { - describe('transformDataToNdjson', () => { - test('if rules are empty it returns an empty string', () => { - const ruleNdjson = transformDataToNdjson([]); - expect(ruleNdjson).toEqual(''); - }); - - test('single rule will transform with new line ending character for ndjson', () => { - const rule = getRulesSchemaMock(); - const ruleNdjson = transformDataToNdjson([rule]); - expect(ruleNdjson.endsWith('\n')).toBe(true); - }); - - test('multiple rules will transform with two new line ending characters for ndjson', () => { - const result1 = getRulesSchemaMock(); - const result2 = getRulesSchemaMock(); - result2.id = 'some other id'; - result2.rule_id = 'some other id'; - result2.name = 'Some other rule'; - - const ruleNdjson = transformDataToNdjson([result1, result2]); - // this is how we count characters in JavaScript :-) - const count = ruleNdjson.split('\n').length - 1; - expect(count).toBe(2); - }); - - test('you can parse two rules back out without errors', () => { - const result1 = getRulesSchemaMock(); - const result2 = getRulesSchemaMock(); - result2.id = 'some other id'; - result2.rule_id = 'some other id'; - result2.name = 'Some other rule'; - - const ruleNdjson = transformDataToNdjson([result1, result2]); - const ruleStrings = ruleNdjson.split('\n'); - const reParsed1 = JSON.parse(ruleStrings[0]); - const reParsed2 = JSON.parse(ruleStrings[1]); - expect(reParsed1).toEqual(result1); - expect(reParsed2).toEqual(result2); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.ts b/x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.ts index 42f1b467ed4c2..eb5abaee8cd3b 100644 --- a/x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.ts +++ b/x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.ts @@ -48,12 +48,3 @@ export const createLimitStream = (limit: number): Transform => { }, }); }; - -export const transformDataToNdjson = (data: unknown[]): string => { - if (data.length !== 0) { - const dataString = data.map((rule) => JSON.stringify(rule)).join('\n'); - return `${dataString}\n`; - } else { - return ''; - } -}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts new file mode 100644 index 0000000000000..d35d34fde5bcc --- /dev/null +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import type { CreateExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { EXCEPTION_LIST_URL, EXCEPTION_LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; + +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + removeExceptionListServerGeneratedProperties, + removeExceptionListItemServerGeneratedProperties, + binaryToString, + deleteAllExceptions, +} from '../../utils'; +import { getCreateExceptionListMinimalSchemaMock } from '../../../../plugins/lists/common/schemas/request/create_exception_list_schema.mock'; +import { getCreateExceptionListItemMinimalSchemaMock } from '../../../../plugins/lists/common/schemas/request/create_exception_list_item_schema.mock'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + const es = getService('es'); + + describe('export_exception_list_route', () => { + describe('exporting exception lists', () => { + afterEach(async () => { + await deleteAllExceptions(es); + }); + + it('should set the response content types to be expected', async () => { + // create an exception list + const { body } = await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListMinimalSchemaMock()) + .expect(200); + + // create an exception list item + await supertest + .post(EXCEPTION_LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListItemMinimalSchemaMock()) + .expect(200); + + await supertest + .post( + `${EXCEPTION_LIST_URL}/_export?id=${body.id}&list_id=${body.list_id}&namespace_type=single` + ) + .set('kbn-xsrf', 'true') + .expect('Content-Disposition', `attachment; filename="${body.list_id}"`) + .expect(200); + }); + + it('should return 404 if given ids that do not exist', async () => { + // create an exception list + await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListMinimalSchemaMock()) + .expect(200); + + // create an exception list item + await supertest + .post(EXCEPTION_LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListItemMinimalSchemaMock()) + .expect(200); + + const { body: exportBody } = await supertest + .post( + `${EXCEPTION_LIST_URL}/_export?id=not_exist&list_id=not_exist&namespace_type=single` + ) + .set('kbn-xsrf', 'true') + .expect(400); + + expect(exportBody).to.eql({ + message: 'exception list with list_id: not_exist does not exist', + status_code: 400, + }); + }); + + it('should export a single list with a list id', async () => { + const { body } = await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListMinimalSchemaMock()) + .expect(200); + + const { body: itemBody } = await supertest + .post(EXCEPTION_LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListItemMinimalSchemaMock()) + .expect(200); + + const { body: exportResult } = await supertest + .post( + `${EXCEPTION_LIST_URL}/_export?id=${body.id}&list_id=${body.list_id}&namespace_type=single` + ) + .set('kbn-xsrf', 'true') + .expect(200) + .parse(binaryToString); + + const exportedItemsToArray = exportResult.toString().split('\n'); + const list = JSON.parse(exportedItemsToArray[0]); + const item = JSON.parse(exportedItemsToArray[1]); + + expect(removeExceptionListServerGeneratedProperties(list)).to.eql( + removeExceptionListServerGeneratedProperties(body) + ); + expect(removeExceptionListItemServerGeneratedProperties(item)).to.eql( + removeExceptionListItemServerGeneratedProperties(itemBody) + ); + }); + + it('should export two list items with a list id', async () => { + const { body } = await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListMinimalSchemaMock()) + .expect(200); + + await supertest + .post(EXCEPTION_LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListItemMinimalSchemaMock()) + .expect(200); + + const secondExceptionListItem: CreateExceptionListItemSchema = { + ...getCreateExceptionListItemMinimalSchemaMock(), + item_id: 'some-list-item-id-2', + }; + await supertest + .post(EXCEPTION_LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(secondExceptionListItem) + .expect(200); + + const { body: exportResult } = await supertest + .post( + `${EXCEPTION_LIST_URL}/_export?id=${body.id}&list_id=${body.list_id}&namespace_type=single` + ) + .set('kbn-xsrf', 'true') + .expect(200) + .parse(binaryToString); + + const bodyString = exportResult.toString(); + expect(bodyString.includes('some-list-item-id-2')).to.be(true); + expect(bodyString.includes('some-list-item-id')).to.be(true); + }); + }); + }); +}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts index 89a1183da6790..afb6057dedfff 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts @@ -24,6 +24,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./find_list_items')); loadTestFile(require.resolve('./import_list_items')); loadTestFile(require.resolve('./export_list_items')); + loadTestFile(require.resolve('./export_exception_list')); loadTestFile(require.resolve('./create_exception_lists')); loadTestFile(require.resolve('./create_exception_list_items')); loadTestFile(require.resolve('./read_exception_lists')); From 95e8595a1287ee9148846f1f9e4707b703c2d65d Mon Sep 17 00:00:00 2001 From: Justin Ibarra Date: Wed, 13 Oct 2021 21:49:07 -0800 Subject: [PATCH 35/35] [Detection Rules] Add 7.16 rules (#114939) --- ...ion_email_powershell_exchange_mailbox.json | 11 +- .../collection_winrar_encryption.json | 11 +- ...ommand_and_control_common_webservices.json | 29 ++- ..._control_dns_directly_to_the_internet.json | 4 +- ...nd_and_control_dns_tunneling_nslookup.json | 11 +- ...download_rar_powershell_from_internet.json | 4 +- .../command_and_control_iexplore_via_com.json | 24 ++- ...ntrol_port_forwarding_added_registry.json} | 23 +-- ...command_and_control_rdp_tunnel_plink.json} | 18 +- ..._and_control_remote_file_copy_scripts.json | 4 +- ...d_control_teamviewer_remote_file_copy.json | 7 +- .../credential_access_cmdline_dump_tool.json | 16 +- ...ess_copy_ntds_sam_volshadowcp_cmdline.json | 11 +- ...cess_domain_backup_dpapi_private_keys.json | 7 +- ...credential_access_dump_registry_hives.json | 16 +- ..._access_kerberoasting_unusual_process.json | 4 +- .../credential_access_kerberosdump_kcc.json | 14 +- ...ial_access_lsass_memdump_file_created.json | 11 +- ..._potential_lsa_memdump_via_mirrordump.json | 55 ++++++ ...redential_access_saved_creds_vaultcmd.json | 14 +- ...e_evasion_clearing_windows_event_logs.json | 11 +- ...vasion_clearing_windows_security_logs.json | 14 +- ...ion_defender_exclusion_via_powershell.json | 4 +- ...ble_windows_firewall_rules_with_netsh.json | 8 +- ...efense_evasion_disabling_windows_logs.json | 11 +- ...vasion_dotnet_compiler_parent_process.json | 15 +- ...n_elasticache_security_group_creation.json | 61 ++++++ ...he_security_group_modified_or_deleted.json | 61 ++++++ ...evasion_enable_inbound_rdp_with_netsh.json | 8 +- ...n_enable_network_discovery_with_netsh.json | 8 +- ...ecution_control_panel_suspicious_args.json | 56 ++++++ ...ecution_msbuild_started_by_office_app.json | 11 +- ...n_execution_msbuild_started_by_script.json | 11 +- ...ion_msbuild_started_by_system_process.json | 11 +- ...ion_execution_msbuild_started_renamed.json | 11 +- ...sion_execution_windefend_unusual_path.json | 7 +- ..._evasion_file_creation_mult_extension.json | 24 ++- ...on_frontdoor_firewall_policy_deletion.json | 60 ++++++ ...sion_hide_encoded_executable_registry.json | 7 +- ...ense_evasion_iis_httplogging_disabled.json | 15 +- .../defense_evasion_installutil_beacon.json | 4 +- ...e_evasion_masquerading_renamed_autoit.json | 11 +- ...vasion_masquerading_trusted_directory.json | 11 +- ...defense_evasion_masquerading_werfault.json | 4 +- ...on_msbuild_making_network_connections.json | 11 +- ...cess_termination_followed_by_deletion.json | 11 +- .../defense_evasion_unusual_dir_ads.json | 11 +- .../defense_evasion_via_filter_manager.json | 15 +- ...on_whitespace_padding_in_command_line.json | 4 +- .../discovery_adfind_command_activity.json | 9 +- .../discovery_admin_recon.json | 11 +- .../discovery_net_command_system_account.json | 8 +- .../discovery_security_software_wmic.json | 11 +- ...y_virtual_machine_fingerprinting_grep.json | 52 ++++++ .../execution_enumeration_via_wmiprvse.json | 27 ++- ...le_program_connecting_to_the_internet.json | 17 +- ...tion_scheduled_task_powershell_source.json | 11 +- .../execution_via_compiled_html_file.json | 17 +- .../exfiltration_rds_snapshot_export.json | 5 +- .../impact_backup_file_deletion.json | 52 ++++++ ...eleting_backup_catalogs_with_wbadmin.json} | 23 +-- ...oft_365_potential_ransomware_activity.json | 54 ++++++ ...t_365_unusual_volume_of_file_deletion.json | 54 ++++++ ...> impact_modification_of_boot_config.json} | 23 +-- ...mpact_stop_process_service_threshold.json} | 25 +-- ...opy_deletion_or_resized_via_vssadmin.json} | 8 +- ...e_shadow_copy_deletion_via_powershell.json | 52 ++++++ ...volume_shadow_copy_deletion_via_wmic.json} | 23 +-- .../rules/prepackaged_rules/index.ts | 174 +++++++++++------- ...65_user_restricted_from_sending_email.json | 54 ++++++ ...ta_user_attempted_unauthorized_access.json | 74 ++++++++ ...ss_suspicious_ms_office_child_process.json | 4 +- ...ential_access_kerberos_bifrostconsole.json | 11 +- .../lateral_movement_dcom_hta.json | 35 +++- .../lateral_movement_dcom_mmc20.json | 13 +- ...t_dcom_shellwindow_shellbrowserwindow.json | 13 +- ...vement_direct_outbound_smb_connection.json | 15 +- .../lateral_movement_dns_server_overflow.json | 4 +- ...movement_executable_tool_transfer_smb.json | 4 +- ...vement_incoming_winrm_shell_execution.json | 4 +- .../lateral_movement_incoming_wmi.json | 4 +- ...l_movement_powershell_remoting_target.json | 4 +- ...lateral_movement_rdp_enabled_registry.json | 11 +- .../lateral_movement_rdp_sharprdp_target.json | 13 +- .../lateral_movement_remote_services.json | 4 +- ...ateral_movement_scheduled_task_target.json | 15 +- .../ml_auth_rare_user_logon.json | 4 +- ...pike_in_logon_events_from_a_source_ip.json | 4 +- .../ml_cloudtrail_error_message_spike.json | 4 +- .../ml_cloudtrail_rare_error_code.json | 4 +- .../ml_cloudtrail_rare_method_by_city.json | 4 +- .../ml_cloudtrail_rare_method_by_country.json | 4 +- .../ml_cloudtrail_rare_method_by_user.json | 4 +- .../ml_rare_process_by_host_windows.json | 4 +- ..._group_configuration_change_detection.json | 19 +- ...evasion_hidden_local_account_creation.json | 11 +- ...egistry_startup_shell_folder_modified.json | 4 +- ...e_suspicious_mailbox_right_delegation.json | 57 ++++++ ...sistence_gpo_schtask_service_creation.json | 11 +- ...sistence_local_scheduled_job_creation.json | 11 +- ...stence_local_scheduled_task_scripting.json | 11 +- ...l_exch_mailbox_activesync_add_device.json} | 28 +-- .../persistence_rds_instance_creation.json | 5 +- .../persistence_registry_uncommon.json | 17 +- ...tence_route_table_modified_or_deleted.json | 55 ++++++ ...saver_engine_unexpected_child_process.json | 50 +++++ ...e_screensaver_plist_file_modification.json | 50 +++++ ...nce_suspicious_scheduled_task_runtime.json | 11 +- ..._account_added_to_privileged_group_ad.json | 15 +- .../persistence_user_account_creation.json | 11 +- ...emetrycontroller_scheduledtask_hijack.json | 11 +- ...nt_instrumentation_event_subscription.json | 11 +- .../persistence_webshell_detection.json | 4 +- ...ion_new_or_modified_federation_domain.json | 61 ++++++ ..._escalation_sts_getsessiontoken_abuse.json | 74 ++++++++ ...tion_unusual_parentchild_relationship.json | 4 +- .../threat_intel_module_match.json | 6 +- 117 files changed, 1920 insertions(+), 377 deletions(-) rename x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/{defense_evasion_port_forwarding_added_registry.json => command_and_control_port_forwarding_added_registry.json} (66%) rename x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/{lateral_movement_rdp_tunnel_plink.json => command_and_control_rdp_tunnel_plink.json} (69%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_potential_lsa_memdump_via_mirrordump.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_elasticache_security_group_creation.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_elasticache_security_group_modified_or_deleted.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_control_panel_suspicious_args.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_frontdoor_firewall_policy_deletion.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting_grep.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_backup_file_deletion.json rename x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/{defense_evasion_deleting_backup_catalogs_with_wbadmin.json => impact_deleting_backup_catalogs_with_wbadmin.json} (66%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_microsoft_365_potential_ransomware_activity.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_microsoft_365_unusual_volume_of_file_deletion.json rename x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/{defense_evasion_modification_of_boot_config.json => impact_modification_of_boot_config.json} (69%) rename x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/{defense_evasion_stop_process_service_threshold.json => impact_stop_process_service_threshold.json} (64%) rename x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/{impact_volume_shadow_copy_deletion_via_vssadmin.json => impact_volume_shadow_copy_deletion_or_resized_via_vssadmin.json} (70%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_powershell.json rename x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/{defense_evasion_volume_shadow_copy_deletion_via_wmic.json => impact_volume_shadow_copy_deletion_via_wmic.json} (66%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_microsoft_365_user_restricted_from_sending_email.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_okta_user_attempted_unauthorized_access.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_exchange_suspicious_mailbox_right_delegation.json rename x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/{collection_persistence_powershell_exch_mailbox_activesync_add_device.json => persistence_powershell_exch_mailbox_activesync_add_device.json} (72%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_route_table_modified_or_deleted.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_screensaver_engine_unexpected_child_process.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_screensaver_plist_file_modification.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_new_or_modified_federation_domain.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sts_getsessiontoken_abuse.json diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_email_powershell_exchange_mailbox.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_email_powershell_exchange_mailbox.json index 25ad15f1b0a51..6e2073bbb82b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_email_powershell_exchange_mailbox.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_email_powershell_exchange_mailbox.json @@ -42,7 +42,14 @@ { "id": "T1114", "name": "Email Collection", - "reference": "https://attack.mitre.org/techniques/T1114/" + "reference": "https://attack.mitre.org/techniques/T1114/", + "subtechnique": [ + { + "id": "T1114.002", + "name": "Remote Email Collection", + "reference": "https://attack.mitre.org/techniques/T1114/002/" + } + ] }, { "id": "T1005", @@ -54,5 +61,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_winrar_encryption.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_winrar_encryption.json index 73c4300556a02..fa0ee2b18bb15 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_winrar_encryption.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_winrar_encryption.json @@ -38,12 +38,19 @@ { "id": "T1560", "name": "Archive Collected Data", - "reference": "https://attack.mitre.org/techniques/T1560/" + "reference": "https://attack.mitre.org/techniques/T1560/", + "subtechnique": [ + { + "id": "T1560.001", + "name": "Archive via Utility", + "reference": "https://attack.mitre.org/techniques/T1560/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_common_webservices.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_common_webservices.json index 0d80e78c556b9..b1774ab3dd052 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_common_webservices.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_common_webservices.json @@ -38,9 +38,36 @@ "reference": "https://attack.mitre.org/techniques/T1102/" } ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0010", + "name": "Exfiltration", + "reference": "https://attack.mitre.org/tactics/TA0010/" + }, + "technique": [ + { + "id": "T1567", + "name": "Exfiltration Over Web Service", + "reference": "https://attack.mitre.org/techniques/T1567/", + "subtechnique": [ + { + "id": "T1567.001", + "name": "Exfiltration to Code Repository", + "reference": "https://attack.mitre.org/techniques/T1567/001/" + }, + { + "id": "T1567.002", + "name": "Exfiltration to Cloud Storage", + "reference": "https://attack.mitre.org/techniques/T1567/002/" + } + ] + } + ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json index 8567b18670301..f57bd65b6d992 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json @@ -2,7 +2,7 @@ "author": [ "Elastic" ], - "description": "This rule detects when an internal network client sends DNS traffic directly to the Internet. This is atypical behavior for a managed network and can be indicative of malware, exfiltration, command and control, or simply misconfiguration. This DNS activity also impacts your organization's ability to provide enterprise monitoring and logging of DNS and it opens your network to a variety of abuses and malicious communications.", + "description": "This rule detects when an internal network client sends DNS traffic directly to the Internet. This is atypical behavior for a managed network and can be indicative of malware, exfiltration, command and control, or simply misconfiguration. This DNS activity also impacts your organization's ability to provide enterprise monitoring and logging of DNS, and it opens your network to a variety of abuses and malicious communications.", "false_positives": [ "Exclude DNS servers from this rule as this is expected behavior. Endpoints usually query local DNS servers defined in their DHCP scopes, but this may be overridden if a user configures their endpoint to use a remote DNS server. This is uncommon in managed enterprise networks because it could break intranet name resolution when split horizon DNS is utilized. Some consumer VPN services and browser plug-ins may send DNS traffic to remote Internet destinations. In that case, such devices or networks can be excluded from this rule when this is expected behavior." ], @@ -45,5 +45,5 @@ ], "timestamp_override": "event.ingested", "type": "query", - "version": 11 + "version": 12 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_tunneling_nslookup.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_tunneling_nslookup.json index 0920f336bab44..29c30f6bc0b49 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_tunneling_nslookup.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_tunneling_nslookup.json @@ -38,7 +38,14 @@ { "id": "T1071", "name": "Application Layer Protocol", - "reference": "https://attack.mitre.org/techniques/T1071/" + "reference": "https://attack.mitre.org/techniques/T1071/", + "subtechnique": [ + { + "id": "T1071.004", + "name": "DNS", + "reference": "https://attack.mitre.org/techniques/T1071/004/" + } + ] } ] } @@ -50,5 +57,5 @@ "value": 15 }, "type": "threshold", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json index 0bcbb0d2d031d..dcca38dd242d8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json @@ -2,7 +2,7 @@ "author": [ "Elastic" ], - "description": "Detects a Roshal Archive (RAR) file or PowerShell script downloaded from the internet by an internal host. Gaining initial access to a system and then downloading encoded or encrypted tools to move laterally is a common practice for adversaries as a way to protect their more valuable tools and TTPs (tactics, techniques, and procedures). This may be atypical behavior for a managed network and can be indicative of malware, exfiltration, or command and control.", + "description": "Detects a Roshal Archive (RAR) file or PowerShell script downloaded from the internet by an internal host. Gaining initial access to a system and then downloading encoded or encrypted tools to move laterally is a common practice for adversaries as a way to protect their more valuable tools and tactics, techniques, and procedures (TTPs). This may be atypical behavior for a managed network and can be indicative of malware, exfiltration, or command and control.", "false_positives": [ "Downloading RAR or PowerShell files from the Internet may be expected for certain systems. This rule should be tailored to either exclude systems as sources or destinations in which this behavior is expected." ], @@ -52,5 +52,5 @@ ], "timestamp_override": "event.ingested", "type": "query", - "version": 7 + "version": 8 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_iexplore_via_com.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_iexplore_via_com.json index 2cfbbc1c5e101..d0039ab4f02d4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_iexplore_via_com.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_iexplore_via_com.json @@ -41,8 +41,30 @@ "reference": "https://attack.mitre.org/techniques/T1071/" } ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0002", + "name": "Execution", + "reference": "https://attack.mitre.org/tactics/TA0002/" + }, + "technique": [ + { + "id": "T1559", + "name": "Inter-Process Communication", + "reference": "https://attack.mitre.org/techniques/T1559/", + "subtechnique": [ + { + "id": "T1559.001", + "name": "Component Object Model", + "reference": "https://attack.mitre.org/techniques/T1559/001/" + } + ] + } + ] } ], "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_port_forwarding_added_registry.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_forwarding_added_registry.json similarity index 66% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_port_forwarding_added_registry.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_forwarding_added_registry.json index cb5c8e87dcae8..65612e6c28f20 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_port_forwarding_added_registry.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_forwarding_added_registry.json @@ -24,33 +24,26 @@ "Host", "Windows", "Threat Detection", - "Defense Evasion" + "Command and Control" ], "threat": [ { "framework": "MITRE ATT&CK", "tactic": { - "id": "TA0005", - "name": "Defense Evasion", - "reference": "https://attack.mitre.org/tactics/TA0005/" + "id": "TA0011", + "name": "Command and Control", + "reference": "https://attack.mitre.org/tactics/TA0011/" }, "technique": [ { - "id": "T1562", - "name": "Impair Defenses", - "reference": "https://attack.mitre.org/techniques/T1562/", - "subtechnique": [ - { - "id": "T1562.001", - "name": "Disable or Modify Tools", - "reference": "https://attack.mitre.org/techniques/T1562/001/" - } - ] + "id": "T1572", + "name": "Protocol Tunneling", + "reference": "https://attack.mitre.org/techniques/T1572/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_tunnel_plink.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_tunnel_plink.json similarity index 69% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_tunnel_plink.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_tunnel_plink.json index dd6bdfa0c37d6..3c89ff7c9ff9a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_tunnel_plink.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_tunnel_plink.json @@ -2,7 +2,7 @@ "author": [ "Elastic" ], - "description": "Identifies potential use of an SSH utility to establish RDP over a reverse SSH Tunnel. This could be indicative of adversary lateral movement to interactively access restricted networks.", + "description": "Identifies potential use of an SSH utility to establish RDP over a reverse SSH Tunnel. This can be used by attackers to enable routing of network packets that would otherwise not reach their intended destination.", "from": "now-9m", "index": [ "logs-endpoint.events.*", @@ -24,26 +24,26 @@ "Host", "Windows", "Threat Detection", - "Lateral Movement" + "Command and Control" ], "threat": [ { "framework": "MITRE ATT&CK", "tactic": { - "id": "TA0008", - "name": "Lateral Movement", - "reference": "https://attack.mitre.org/tactics/TA0008/" + "id": "TA0011", + "name": "Command and Control", + "reference": "https://attack.mitre.org/tactics/TA0011/" }, "technique": [ { - "id": "T1021", - "name": "Remote Services", - "reference": "https://attack.mitre.org/techniques/T1021/" + "id": "T1572", + "name": "Protocol Tunneling", + "reference": "https://attack.mitre.org/techniques/T1572/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_scripts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_scripts.json index 428b08891c15a..eed29634daeef 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_scripts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_scripts.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Remote File Download via Script Interpreter", - "query": "sequence by host.id, process.entity_id\n [network where process.name : (\"wscript.exe\", \"cscript.exe\") and network.protocol != \"dns\" and\n network.direction == \"outgoing\" and network.type == \"ipv4\" and destination.ip != \"127.0.0.1\"\n ]\n [file where event.type == \"creation\" and file.extension : (\"exe\", \"dll\")]\n", + "query": "sequence by host.id, process.entity_id\n [network where process.name : (\"wscript.exe\", \"cscript.exe\") and network.protocol != \"dns\" and\n network.direction : (\"outgoing\", \"egress\") and network.type == \"ipv4\" and destination.ip != \"127.0.0.1\"\n ]\n [file where event.type == \"creation\" and file.extension : (\"exe\", \"dll\")]\n", "risk_score": 47, "rule_id": "1d276579-3380-4095-ad38-e596a01bc64f", "severity": "medium", @@ -41,5 +41,5 @@ } ], "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json index 08d4df2556f6a..a1f0f061a69bc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json @@ -39,11 +39,16 @@ "id": "T1105", "name": "Ingress Tool Transfer", "reference": "https://attack.mitre.org/techniques/T1105/" + }, + { + "id": "T1219", + "name": "Remote Access Software", + "reference": "https://attack.mitre.org/techniques/T1219/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_cmdline_dump_tool.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_cmdline_dump_tool.json index 32c271f736e4a..9671f3c4edf2a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_cmdline_dump_tool.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_cmdline_dump_tool.json @@ -38,12 +38,24 @@ { "id": "T1003", "name": "OS Credential Dumping", - "reference": "https://attack.mitre.org/techniques/T1003/" + "reference": "https://attack.mitre.org/techniques/T1003/", + "subtechnique": [ + { + "id": "T1003.001", + "name": "LSASS Memory", + "reference": "https://attack.mitre.org/techniques/T1003/001/" + }, + { + "id": "T1003.003", + "name": "NTDS", + "reference": "https://attack.mitre.org/techniques/T1003/003/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_copy_ntds_sam_volshadowcp_cmdline.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_copy_ntds_sam_volshadowcp_cmdline.json index 91613078c6167..0aeba88224138 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_copy_ntds_sam_volshadowcp_cmdline.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_copy_ntds_sam_volshadowcp_cmdline.json @@ -41,12 +41,19 @@ { "id": "T1003", "name": "OS Credential Dumping", - "reference": "https://attack.mitre.org/techniques/T1003/" + "reference": "https://attack.mitre.org/techniques/T1003/", + "subtechnique": [ + { + "id": "T1003.002", + "name": "Security Account Manager", + "reference": "https://attack.mitre.org/techniques/T1003/002/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json index c031fcbf464b1..43ea1078d1583 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json @@ -48,11 +48,16 @@ "reference": "https://attack.mitre.org/techniques/T1552/004/" } ] + }, + { + "id": "T1555", + "name": "Credentials from Password Stores", + "reference": "https://attack.mitre.org/techniques/T1555/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 5 + "version": 6 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_dump_registry_hives.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_dump_registry_hives.json index c3868162cc839..10c6996fa56aa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_dump_registry_hives.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_dump_registry_hives.json @@ -38,12 +38,24 @@ { "id": "T1003", "name": "OS Credential Dumping", - "reference": "https://attack.mitre.org/techniques/T1003/" + "reference": "https://attack.mitre.org/techniques/T1003/", + "subtechnique": [ + { + "id": "T1003.002", + "name": "Security Account Manager", + "reference": "https://attack.mitre.org/techniques/T1003/002/" + }, + { + "id": "T1003.004", + "name": "LSA Secrets", + "reference": "https://attack.mitre.org/techniques/T1003/004/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberoasting_unusual_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberoasting_unusual_process.json index b05ddd7bcc8a2..8fc7cd7b379b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberoasting_unusual_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberoasting_unusual_process.json @@ -15,7 +15,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Kerberos Traffic from Unusual Process", - "query": "network where event.type == \"start\" and network.direction == \"outgoing\" and\n destination.port == 88 and source.port >= 49152 and\n process.executable != \"C:\\\\Windows\\\\System32\\\\lsass.exe\" and destination.address !=\"127.0.0.1\" and destination.address !=\"::1\" and\n /* insert False Positives here */\n not process.name in (\"swi_fc.exe\", \"fsIPcam.exe\", \"IPCamera.exe\", \"MicrosoftEdgeCP.exe\", \"MicrosoftEdge.exe\", \"iexplore.exe\", \"chrome.exe\", \"msedge.exe\", \"opera.exe\", \"firefox.exe\")\n", + "query": "network where event.type == \"start\" and network.direction : (\"outgoing\", \"egress\") and\n destination.port == 88 and source.port >= 49152 and\n process.executable != \"C:\\\\Windows\\\\System32\\\\lsass.exe\" and destination.address !=\"127.0.0.1\" and destination.address !=\"::1\" and\n /* insert False Positives here */\n not process.name in (\"swi_fc.exe\", \"fsIPcam.exe\", \"IPCamera.exe\", \"MicrosoftEdgeCP.exe\", \"MicrosoftEdge.exe\", \"iexplore.exe\", \"chrome.exe\", \"msedge.exe\", \"opera.exe\", \"firefox.exe\")\n", "risk_score": 47, "rule_id": "897dc6b5-b39f-432a-8d75-d3730d50c782", "severity": "medium", @@ -45,5 +45,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json index de5a9d80ed3df..3338895f30feb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json @@ -39,11 +39,23 @@ "id": "T1003", "name": "OS Credential Dumping", "reference": "https://attack.mitre.org/techniques/T1003/" + }, + { + "id": "T1558", + "name": "Steal or Forge Kerberos Tickets", + "reference": "https://attack.mitre.org/techniques/T1558/", + "subtechnique": [ + { + "id": "T1558.003", + "name": "Kerberoasting", + "reference": "https://attack.mitre.org/techniques/T1558/003/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_lsass_memdump_file_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_lsass_memdump_file_created.json index 36b614c628b19..d083fb322e895 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_lsass_memdump_file_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_lsass_memdump_file_created.json @@ -39,12 +39,19 @@ { "id": "T1003", "name": "OS Credential Dumping", - "reference": "https://attack.mitre.org/techniques/T1003/" + "reference": "https://attack.mitre.org/techniques/T1003/", + "subtechnique": [ + { + "id": "T1003.001", + "name": "LSASS Memory", + "reference": "https://attack.mitre.org/techniques/T1003/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_potential_lsa_memdump_via_mirrordump.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_potential_lsa_memdump_via_mirrordump.json new file mode 100644 index 0000000000000..1024d7f3461f5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_potential_lsa_memdump_via_mirrordump.json @@ -0,0 +1,55 @@ +{ + "author": [ + "Elastic" + ], + "description": "Identifies suspicious access to an LSASS handle via DuplicateHandle from an unknown call trace module. This may indicate an attempt to bypass the NtOpenProcess API to evade detection and dump Lsass memory for credential access.", + "from": "now-9m", + "index": [ + "winlogbeat-*", + "logs-windows.*" + ], + "language": "eql", + "license": "Elastic License v2", + "name": "Potential Credential Access via DuplicateHandle in LSASS", + "query": "process where event.code == \"10\" and \n\n /* LSASS requesting DuplicateHandle access right to another process */\n process.name : \"lsass.exe\" and winlog.event_data.GrantedAccess == \"0x40\" and\n\n /* call is coming from an unknown executable region */\n winlog.event_data.CallTrace : \"*UNKNOWN*\"\n", + "references": [ + "https://github.com/CCob/MirrorDump" + ], + "risk_score": 47, + "rule_id": "02a4576a-7480-4284-9327-548a806b5e48", + "severity": "medium", + "tags": [ + "Elastic", + "Host", + "Windows", + "Threat Detection", + "Credential Access" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0006", + "name": "Credential Access", + "reference": "https://attack.mitre.org/tactics/TA0006/" + }, + "technique": [ + { + "id": "T1003", + "name": "OS Credential Dumping", + "reference": "https://attack.mitre.org/techniques/T1003/", + "subtechnique": [ + { + "id": "T1003.001", + "name": "LSASS Memory", + "reference": "https://attack.mitre.org/techniques/T1003/001/" + } + ] + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_saved_creds_vaultcmd.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_saved_creds_vaultcmd.json index bfb4e44d39b0d..c6db4426ac8c1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_saved_creds_vaultcmd.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_saved_creds_vaultcmd.json @@ -40,11 +40,23 @@ "id": "T1003", "name": "OS Credential Dumping", "reference": "https://attack.mitre.org/techniques/T1003/" + }, + { + "id": "T1555", + "name": "Credentials from Password Stores", + "reference": "https://attack.mitre.org/techniques/T1555/", + "subtechnique": [ + { + "id": "T1555.004", + "name": "Windows Credential Manager", + "reference": "https://attack.mitre.org/techniques/T1555/004/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json index 79e059d68a52a..2759055b0fe5b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json @@ -35,12 +35,19 @@ { "id": "T1070", "name": "Indicator Removal on Host", - "reference": "https://attack.mitre.org/techniques/T1070/" + "reference": "https://attack.mitre.org/techniques/T1070/", + "subtechnique": [ + { + "id": "T1070.001", + "name": "Clear Windows Event Logs", + "reference": "https://attack.mitre.org/techniques/T1070/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 9 + "version": 10 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_security_logs.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_security_logs.json index d04c2b2a38915..eedca883e371c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_security_logs.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_security_logs.json @@ -7,7 +7,8 @@ "from": "now-9m", "index": [ "winlogbeat-*", - "logs-windows.*" + "logs-windows.*", + "logs-system.*" ], "language": "kuery", "license": "Elastic License v2", @@ -35,12 +36,19 @@ { "id": "T1070", "name": "Indicator Removal on Host", - "reference": "https://attack.mitre.org/techniques/T1070/" + "reference": "https://attack.mitre.org/techniques/T1070/", + "subtechnique": [ + { + "id": "T1070.001", + "name": "Clear Windows Event Logs", + "reference": "https://attack.mitre.org/techniques/T1070/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "query", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_defender_exclusion_via_powershell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_defender_exclusion_via_powershell.json index 000384eac660e..716040d337c10 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_defender_exclusion_via_powershell.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_defender_exclusion_via_powershell.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Windows Defender Exclusions Added via PowerShell", - "note": "## Triage and analysis\n\nDetections should be investigated to identify if the activity corresponds to legitimate activity used to put in exceptions for Windows Defender. As this rule detects post-exploitation process activity, investigations into this should be prioritized.", + "note": "## Triage and analysis\n\n### Investigating Windows Defender Exclusions\n\nMicrosoft Windows Defender is an anti-virus product built-in within Microsoft Windows. Since this software product is\nused to prevent and stop malware, it's important to monitor what specific exclusions are made to the product's configuration\nsettings. These can often be signs of an adversary or malware trying to bypass Windows Defender's capabilities. One of the more\nnotable [examples](https://www.cyberbit.com/blog/endpoint-security/latest-trickbot-variant-has-new-tricks-up-its-sleeve/) was observed in 2018 where Trickbot incorporated mechanisms to disable Windows Defense to avoid detection.\n\n#### Possible investigation steps:\n- With this specific rule, it's completely possible to trigger detections on network administrative activity or benign users\nusing scripting and PowerShell to configure the different exclusions for Windows Defender. Therefore, it's important to\nidentify the source of the activity first and determine if there is any mal-intent behind the events.\n- The actual exclusion such as the process, the file or directory should be reviewed in order to determine the original\nintent behind the exclusion. Is the excluded file or process malicious in nature or is it related to software that needs\nto be legitimately whitelisted from Windows Defender?\n\n### False Positive Analysis\n- This rule has a higher chance to produce false positives based on the nature around configuring exclusions by possibly\na network administrator. In order to validate the activity further, review the specific exclusion made and determine based\non the exclusion of the original intent behind the exclusion. There are often many legitimate reasons why exclusions are made\nwith Windows Defender so it's important to gain context around the exclusion.\n\n### Related Rules\n- Windows Defender Disabled via Registry Modification\n- Disabling Windows Defender Security Settings via PowerShell\n\n### Response and Remediation\n- Since this is related to post-exploitation activity, immediate response should be taken to review, investigate and\npotentially isolate further activity\n- If further analysis showed malicious intent was behind the Defender exclusions, administrators should remove\nthe exclusion and ensure antimalware capability has not been disabled or deleted\n- Exclusion lists for antimalware capabilities should always be routinely monitored for review\n", "query": "process where event.type == \"start\" and\n (process.name : (\"powershell.exe\", \"pwsh.exe\") or process.pe.original_file_name : (\"powershell.exe\", \"pwsh.exe\")) and\n process.args : (\"*Add-MpPreference*-Exclusion*\", \"*Set-MpPreference*-Exclusion*\")\n", "references": [ "https://www.bitdefender.com/files/News/CaseStudies/study/400/Bitdefender-PR-Whitepaper-MosaicLoader-creat5540-en-EN.pdf" @@ -80,5 +80,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json index 00f18df34f864..2e18f3ba62786 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json @@ -38,9 +38,9 @@ "reference": "https://attack.mitre.org/techniques/T1562/", "subtechnique": [ { - "id": "T1562.001", - "name": "Disable or Modify Tools", - "reference": "https://attack.mitre.org/techniques/T1562/001/" + "id": "T1562.004", + "name": "Disable or Modify System Firewall", + "reference": "https://attack.mitre.org/techniques/T1562/004/" } ] } @@ -49,5 +49,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 9 + "version": 10 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disabling_windows_logs.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disabling_windows_logs.json index d2612101a3e4c..256d1c7d9c135 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disabling_windows_logs.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disabling_windows_logs.json @@ -40,12 +40,19 @@ { "id": "T1070", "name": "Indicator Removal on Host", - "reference": "https://attack.mitre.org/techniques/T1070/" + "reference": "https://attack.mitre.org/techniques/T1070/", + "subtechnique": [ + { + "id": "T1070.001", + "name": "Clear Windows Event Logs", + "reference": "https://attack.mitre.org/techniques/T1070/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json index 4588a8ab28657..e8edb8fba6472 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json @@ -33,14 +33,21 @@ }, "technique": [ { - "id": "T1055", - "name": "Process Injection", - "reference": "https://attack.mitre.org/techniques/T1055/" + "id": "T1027", + "name": "Obfuscated Files or Information", + "reference": "https://attack.mitre.org/techniques/T1027/", + "subtechnique": [ + { + "id": "T1027.004", + "name": "Compile After Delivery", + "reference": "https://attack.mitre.org/techniques/T1027/004/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_elasticache_security_group_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_elasticache_security_group_creation.json new file mode 100644 index 0000000000000..5685ac76b3ef9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_elasticache_security_group_creation.json @@ -0,0 +1,61 @@ +{ + "author": [ + "Austin Songer" + ], + "description": "Identifies when an ElastiCache security group has been created.", + "false_positives": [ + "A ElastiCache security group may be created by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. Security group creations from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule." + ], + "from": "now-60m", + "index": [ + "filebeat-*", + "logs-aws*" + ], + "interval": "10m", + "language": "kuery", + "license": "Elastic License v2", + "name": "AWS ElastiCache Security Group Created", + "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.", + "query": "event.dataset:aws.cloudtrail and event.provider:elasticache.amazonaws.com and event.action:\"Create Cache Security Group\" and \nevent.outcome:success\n", + "references": [ + "https://docs.aws.amazon.com/AmazonElastiCache/latest/APIReference/API_CreateCacheSecurityGroup.html" + ], + "risk_score": 21, + "rule_id": "7b3da11a-60a2-412e-8aa7-011e1eb9ed47", + "severity": "low", + "tags": [ + "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", + "SecOps", + "Monitoring" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0005", + "name": "Defense Evasion", + "reference": "https://attack.mitre.org/tactics/TA0005/" + }, + "technique": [ + { + "id": "T1562", + "name": "Impair Defenses", + "reference": "https://attack.mitre.org/techniques/T1562/", + "subtechnique": [ + { + "id": "T1562.007", + "name": "Disable or Modify Cloud Firewall", + "reference": "https://attack.mitre.org/techniques/T1562/007/" + } + ] + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_elasticache_security_group_modified_or_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_elasticache_security_group_modified_or_deleted.json new file mode 100644 index 0000000000000..83b58c0c046e0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_elasticache_security_group_modified_or_deleted.json @@ -0,0 +1,61 @@ +{ + "author": [ + "Austin Songer" + ], + "description": "Identifies when an ElastiCache security group has been modified or deleted.", + "false_positives": [ + "A ElastiCache security group deletion may be done by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. Security Group deletions from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule." + ], + "from": "now-60m", + "index": [ + "filebeat-*", + "logs-aws*" + ], + "interval": "10m", + "language": "kuery", + "license": "Elastic License v2", + "name": "AWS ElastiCache Security Group Modified or Deleted", + "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.", + "query": "event.dataset:aws.cloudtrail and event.provider:elasticache.amazonaws.com and event.action:(\"Delete Cache Security Group\" or \n\"Authorize Cache Security Group Ingress\" or \"Revoke Cache Security Group Ingress\" or \"AuthorizeCacheSecurityGroupEgress\" or \n\"RevokeCacheSecurityGroupEgress\") and event.outcome:success\n", + "references": [ + "https://docs.aws.amazon.com/AmazonElastiCache/latest/APIReference/Welcome.html" + ], + "risk_score": 21, + "rule_id": "1ba5160d-f5a2-4624-b0ff-6a1dc55d2516", + "severity": "low", + "tags": [ + "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", + "SecOps", + "Monitoring" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0005", + "name": "Defense Evasion", + "reference": "https://attack.mitre.org/tactics/TA0005/" + }, + "technique": [ + { + "id": "T1562", + "name": "Impair Defenses", + "reference": "https://attack.mitre.org/techniques/T1562/", + "subtechnique": [ + { + "id": "T1562.007", + "name": "Disable or Modify Cloud Firewall", + "reference": "https://attack.mitre.org/techniques/T1562/007/" + } + ] + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_enable_inbound_rdp_with_netsh.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_enable_inbound_rdp_with_netsh.json index 93454122d1160..e6b53af71433a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_enable_inbound_rdp_with_netsh.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_enable_inbound_rdp_with_netsh.json @@ -38,9 +38,9 @@ "reference": "https://attack.mitre.org/techniques/T1562/", "subtechnique": [ { - "id": "T1562.001", - "name": "Disable or Modify Tools", - "reference": "https://attack.mitre.org/techniques/T1562/001/" + "id": "T1562.004", + "name": "Disable or Modify System Firewall", + "reference": "https://attack.mitre.org/techniques/T1562/004/" } ] } @@ -49,5 +49,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_enable_network_discovery_with_netsh.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_enable_network_discovery_with_netsh.json index 5fcbec498a177..bf688fd74ce14 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_enable_network_discovery_with_netsh.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_enable_network_discovery_with_netsh.json @@ -41,9 +41,9 @@ "reference": "https://attack.mitre.org/techniques/T1562/", "subtechnique": [ { - "id": "T1562.001", - "name": "Disable or Modify Tools", - "reference": "https://attack.mitre.org/techniques/T1562/001/" + "id": "T1562.004", + "name": "Disable or Modify System Firewall", + "reference": "https://attack.mitre.org/techniques/T1562/004/" } ] } @@ -52,5 +52,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_control_panel_suspicious_args.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_control_panel_suspicious_args.json new file mode 100644 index 0000000000000..787e61cfe25c4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_control_panel_suspicious_args.json @@ -0,0 +1,56 @@ +{ + "author": [ + "Elastic" + ], + "description": "Identifies unusual instances of Control Panel with suspicious keywords or paths in the process command line value. Adversaries may abuse Control.exe to proxy execution of malicious code.", + "from": "now-9m", + "index": [ + "logs-endpoint.events.*", + "winlogbeat-*", + "logs-windows.*" + ], + "language": "eql", + "license": "Elastic License v2", + "name": "Control Panel Process with Unusual Arguments", + "query": "process where event.type in (\"start\", \"process_started\") and\n process.executable : (\"?:\\\\Windows\\\\SysWOW64\\\\control.exe\", \"?:\\\\Windows\\\\System32\\\\control.exe\") and\n process.command_line :\n (\"*.jpg*\",\n \"*.png*\",\n \"*.gif*\",\n \"*.bmp*\",\n \"*.jpeg*\",\n \"*.TIFF*\",\n \"*.inf*\",\n \"*.dat*\",\n \"*.cpl:*/*\",\n \"*../../..*\",\n \"*/AppData/Local/*\",\n \"*:\\\\Users\\\\Public\\\\*\",\n \"*\\\\AppData\\\\Local\\\\*\")\n", + "references": [ + "https://www.joesandbox.com/analysis/476188/1/html" + ], + "risk_score": 73, + "rule_id": "416697ae-e468-4093-a93d-59661fa619ec", + "severity": "high", + "tags": [ + "Elastic", + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0005", + "name": "Defense Evasion", + "reference": "https://attack.mitre.org/tactics/TA0005/" + }, + "technique": [ + { + "id": "T1218", + "name": "Signed Binary Proxy Execution", + "reference": "https://attack.mitre.org/techniques/T1218/", + "subtechnique": [ + { + "id": "T1218.002", + "name": "Control Panel", + "reference": "https://attack.mitre.org/techniques/T1218/002/" + } + ] + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json index d56c90552d457..0ad45f03a0499 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json @@ -41,7 +41,14 @@ { "id": "T1127", "name": "Trusted Developer Utilities Proxy Execution", - "reference": "https://attack.mitre.org/techniques/T1127/" + "reference": "https://attack.mitre.org/techniques/T1127/", + "subtechnique": [ + { + "id": "T1127.001", + "name": "MSBuild", + "reference": "https://attack.mitre.org/techniques/T1127/001/" + } + ] } ] }, @@ -57,5 +64,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 8 + "version": 9 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json index 3b640d8757b51..60b2a8f50c3f4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json @@ -38,7 +38,14 @@ { "id": "T1127", "name": "Trusted Developer Utilities Proxy Execution", - "reference": "https://attack.mitre.org/techniques/T1127/" + "reference": "https://attack.mitre.org/techniques/T1127/", + "subtechnique": [ + { + "id": "T1127.001", + "name": "MSBuild", + "reference": "https://attack.mitre.org/techniques/T1127/001/" + } + ] } ] }, @@ -54,5 +61,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 8 + "version": 9 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json index 33094a88af313..fdee8ee548218 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json @@ -38,7 +38,14 @@ { "id": "T1127", "name": "Trusted Developer Utilities Proxy Execution", - "reference": "https://attack.mitre.org/techniques/T1127/" + "reference": "https://attack.mitre.org/techniques/T1127/", + "subtechnique": [ + { + "id": "T1127.001", + "name": "MSBuild", + "reference": "https://attack.mitre.org/techniques/T1127/001/" + } + ] } ] }, @@ -54,5 +61,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 8 + "version": 9 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json index 43051cb8b27c9..a22594083bedb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json @@ -38,12 +38,19 @@ { "id": "T1036", "name": "Masquerading", - "reference": "https://attack.mitre.org/techniques/T1036/" + "reference": "https://attack.mitre.org/techniques/T1036/", + "subtechnique": [ + { + "id": "T1036.003", + "name": "Rename System Utilities", + "reference": "https://attack.mitre.org/techniques/T1036/003/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 8 + "version": 9 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_windefend_unusual_path.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_windefend_unusual_path.json index 7812dee8235ca..826d55f3b1882 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_windefend_unusual_path.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_windefend_unusual_path.json @@ -1,6 +1,7 @@ { "author": [ - "Elastic" + "Elastic", + "Dennis Perto" ], "description": "Identifies a Windows trusted program that is known to be vulnerable to DLL Search Order Hijacking starting after being renamed or from a non-standard path. This is uncommon behavior and may indicate an attempt to evade defenses via side-loading a malicious DLL within the memory space of one of those processes.", "false_positives": [ @@ -15,7 +16,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Potential DLL Side-Loading via Microsoft Antimalware Service Executable", - "query": "process where event.type == \"start\" and\n (process.pe.original_file_name == \"MsMpEng.exe\" and not process.name : \"MsMpEng.exe\") or\n (process.name : \"MsMpEng.exe\" and not\n process.executable : (\"?:\\\\ProgramData\\\\Microsoft\\\\Windows Defender\\\\*.exe\",\n \"?:\\\\Program Files\\\\Windows Defender\\\\*.exe\",\n \"?:\\\\Program Files (x86)\\\\Windows Defender\\\\*.exe\"))\n", + "query": "process where event.type == \"start\" and\n (process.pe.original_file_name == \"MsMpEng.exe\" and not process.name : \"MsMpEng.exe\") or\n (process.name : \"MsMpEng.exe\" and not\n process.executable : (\"?:\\\\ProgramData\\\\Microsoft\\\\Windows Defender\\\\*.exe\",\n \"?:\\\\Program Files\\\\Windows Defender\\\\*.exe\",\n \"?:\\\\Program Files (x86)\\\\Windows Defender\\\\*.exe\",\n \"?:\\\\Program Files\\\\Microsoft Security Client\\\\*.exe\",\n \"?:\\\\Program Files (x86)\\\\Microsoft Security Client\\\\*.exe\"))\n", "references": [ "https://news.sophos.com/en-us/2021/07/04/independence-day-revil-uses-supply-chain-exploit-to-attack-hundreds-of-businesses/" ], @@ -55,5 +56,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_creation_mult_extension.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_creation_mult_extension.json index 24cbb1e41dad6..4cbfb8bbbce6c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_creation_mult_extension.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_creation_mult_extension.json @@ -45,9 +45,31 @@ ] } ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0002", + "name": "Execution", + "reference": "https://attack.mitre.org/tactics/TA0002/" + }, + "technique": [ + { + "id": "T1204", + "name": "User Execution", + "reference": "https://attack.mitre.org/techniques/T1204/", + "subtechnique": [ + { + "id": "T1204.002", + "name": "Malicious File", + "reference": "https://attack.mitre.org/techniques/T1204/002/" + } + ] + } + ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_frontdoor_firewall_policy_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_frontdoor_firewall_policy_deletion.json new file mode 100644 index 0000000000000..c443d45dde4f0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_frontdoor_firewall_policy_deletion.json @@ -0,0 +1,60 @@ +{ + "author": [ + "Austin Songer" + ], + "description": "Identifies the deletion of a Frontdoor Web Application Firewall (WAF) Policy in Azure. An adversary may delete a Frontdoor Web Application Firewall (WAF) Policy in an attempt to evade defenses and/or to eliminate barriers in carrying out their initiative.", + "false_positives": [ + "Azure Front Web Application Firewall (WAF) Policy deletions may be done by a system or network administrator. Verify whether the username, hostname, and/or resource name should be making changes in your environment. Azure Front Web Application Firewall (WAF) Policy deletions from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule." + ], + "from": "now-25m", + "index": [ + "filebeat-*", + "logs-azure*" + ], + "language": "kuery", + "license": "Elastic License v2", + "name": "Azure Frontdoor Web Application Firewall (WAF) Policy Deleted", + "note": "## Config\n\nThe Azure Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.", + "query": "event.dataset:azure.activitylogs and azure.activitylogs.operation_name:\"MICROSOFT.NETWORK/FRONTDOORWEBAPPLICATIONFIREWALLPOLICIES/DELETE\" and event.outcome:(Success or success)\n", + "references": [ + "https://docs.microsoft.com/en-us/azure/role-based-access-control/resource-provider-operations#networking" + ], + "risk_score": 21, + "rule_id": "09d028a5-dcde-409f-8ae0-557cef1b7082", + "severity": "low", + "tags": [ + "Elastic", + "Cloud", + "Azure", + "Continuous Monitoring", + "SecOps", + "Network Security" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0005", + "name": "Defense Evasion", + "reference": "https://attack.mitre.org/tactics/TA0005/" + }, + "technique": [ + { + "id": "T1562", + "name": "Impair Defenses", + "reference": "https://attack.mitre.org/techniques/T1562/", + "subtechnique": [ + { + "id": "T1562.001", + "name": "Disable or Modify Tools", + "reference": "https://attack.mitre.org/techniques/T1562/001/" + } + ] + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hide_encoded_executable_registry.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hide_encoded_executable_registry.json index 006f95054d047..c40bbf236d668 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hide_encoded_executable_registry.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hide_encoded_executable_registry.json @@ -36,11 +36,16 @@ "id": "T1140", "name": "Deobfuscate/Decode Files or Information", "reference": "https://attack.mitre.org/techniques/T1140/" + }, + { + "id": "T1112", + "name": "Modify Registry", + "reference": "https://attack.mitre.org/techniques/T1112/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json index 16de1c9c21f97..da12646d40226 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json @@ -34,14 +34,21 @@ }, "technique": [ { - "id": "T1070", - "name": "Indicator Removal on Host", - "reference": "https://attack.mitre.org/techniques/T1070/" + "id": "T1562", + "name": "Impair Defenses", + "reference": "https://attack.mitre.org/techniques/T1562/", + "subtechnique": [ + { + "id": "T1562.002", + "name": "Disable Windows Event Logging", + "reference": "https://attack.mitre.org/techniques/T1562/002/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 5 + "version": 6 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json index 4917cffd64ccb..72ef939fd2c1c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "InstallUtil Process Making Network Connections", - "query": "/* the benefit of doing this as an eql sequence vs kql is this will limit to alerting only on the first network connection */\n\nsequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name : \"installutil.exe\"]\n [network where process.name : \"installutil.exe\" and network.direction == \"outgoing\"]\n", + "query": "/* the benefit of doing this as an eql sequence vs kql is this will limit to alerting only on the first network connection */\n\nsequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name : \"installutil.exe\"]\n [network where process.name : \"installutil.exe\" and network.direction : (\"outgoing\", \"egress\")]\n", "risk_score": 21, "rule_id": "a13167f1-eec2-4015-9631-1fee60406dcf", "severity": "medium", @@ -48,5 +48,5 @@ } ], "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json index bd0a3ac9f918d..5c855207dda7d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json @@ -35,12 +35,19 @@ { "id": "T1036", "name": "Masquerading", - "reference": "https://attack.mitre.org/techniques/T1036/" + "reference": "https://attack.mitre.org/techniques/T1036/", + "subtechnique": [ + { + "id": "T1036.003", + "name": "Rename System Utilities", + "reference": "https://attack.mitre.org/techniques/T1036/003/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_trusted_directory.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_trusted_directory.json index b0d11121c1a15..7ac21a70100c0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_trusted_directory.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_trusted_directory.json @@ -35,12 +35,19 @@ { "id": "T1036", "name": "Masquerading", - "reference": "https://attack.mitre.org/techniques/T1036/" + "reference": "https://attack.mitre.org/techniques/T1036/", + "subtechnique": [ + { + "id": "T1036.005", + "name": "Match Legitimate Name or Location", + "reference": "https://attack.mitre.org/techniques/T1036/005/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json index 2733bf992838e..a08e3040c6c95 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json @@ -15,7 +15,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Potential Windows Error Manager Masquerading", - "query": "sequence by host.id, process.entity_id with maxspan = 5s\n [process where event.type:\"start\" and process.name : (\"wermgr.exe\", \"WerFault.exe\") and process.args_count == 1]\n [network where process.name : (\"wermgr.exe\", \"WerFault.exe\") and network.protocol != \"dns\" and\n network.direction == \"outgoing\" and destination.ip !=\"::1\" and destination.ip !=\"127.0.0.1\"\n ]\n", + "query": "sequence by host.id, process.entity_id with maxspan = 5s\n [process where event.type:\"start\" and process.name : (\"wermgr.exe\", \"WerFault.exe\") and process.args_count == 1]\n [network where process.name : (\"wermgr.exe\", \"WerFault.exe\") and network.protocol != \"dns\" and\n network.direction : (\"outgoing\", \"egress\") and destination.ip !=\"::1\" and destination.ip !=\"127.0.0.1\"\n ]\n", "references": [ "https://twitter.com/SBousseaden/status/1235533224337641473", "https://www.hexacorn.com/blog/2019/09/20/werfault-command-line-switches-v0-1/", @@ -49,5 +49,5 @@ } ], "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_making_network_connections.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_making_network_connections.json index a2019165f93c6..6d0110c229c33 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_making_network_connections.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_making_network_connections.json @@ -35,11 +35,18 @@ { "id": "T1127", "name": "Trusted Developer Utilities Proxy Execution", - "reference": "https://attack.mitre.org/techniques/T1127/" + "reference": "https://attack.mitre.org/techniques/T1127/", + "subtechnique": [ + { + "id": "T1127.001", + "name": "MSBuild", + "reference": "https://attack.mitre.org/techniques/T1127/001/" + } + ] } ] } ], "type": "eql", - "version": 7 + "version": 8 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_process_termination_followed_by_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_process_termination_followed_by_deletion.json index b7d65b2336001..85316f7836b89 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_process_termination_followed_by_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_process_termination_followed_by_deletion.json @@ -33,11 +33,18 @@ { "id": "T1070", "name": "Indicator Removal on Host", - "reference": "https://attack.mitre.org/techniques/T1070/" + "reference": "https://attack.mitre.org/techniques/T1070/", + "subtechnique": [ + { + "id": "T1070.004", + "name": "File Deletion", + "reference": "https://attack.mitre.org/techniques/T1070/004/" + } + ] } ] } ], "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_dir_ads.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_dir_ads.json index 196a3de9b9e6f..f926a1ba24faf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_dir_ads.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_dir_ads.json @@ -35,12 +35,19 @@ { "id": "T1564", "name": "Hide Artifacts", - "reference": "https://attack.mitre.org/techniques/T1564/" + "reference": "https://attack.mitre.org/techniques/T1564/", + "subtechnique": [ + { + "id": "T1564.004", + "name": "NTFS File Attributes", + "reference": "https://attack.mitre.org/techniques/T1564/004/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json index 51d1789804548..c0d171739b76d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json @@ -33,14 +33,21 @@ }, "technique": [ { - "id": "T1222", - "name": "File and Directory Permissions Modification", - "reference": "https://attack.mitre.org/techniques/T1222/" + "id": "T1562", + "name": "Impair Defenses", + "reference": "https://attack.mitre.org/techniques/T1562/", + "subtechnique": [ + { + "id": "T1562.001", + "name": "Disable or Modify Tools", + "reference": "https://attack.mitre.org/techniques/T1562/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 7 + "version": 8 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_whitespace_padding_in_command_line.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_whitespace_padding_in_command_line.json index fc9b480023c95..f022f0c27ff5e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_whitespace_padding_in_command_line.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_whitespace_padding_in_command_line.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Whitespace Padding in Process Command Line", - "note": "## Triage and analysis\n\n- Analyze the command line of the process in question for evidence of malicious code execution.\n- Review the ancestry and child processes spawned by the process in question for indicators of further malicious code execution.", + "note": "## Triage and analysis\n\n- Analyze the command line of the process in question for evidence of malicious code execution.\n- Review the ancestor and child processes spawned by the process in question for indicators of further malicious code execution.", "query": "process where event.type in (\"start\", \"process_started\") and\n process.command_line regex \".*[ ]{20,}.*\" or \n \n /* this will match on 3 or more separate occurrences of 5+ contiguous whitespace characters */\n process.command_line regex \".*(.*[ ]{5,}[^ ]*){3,}.*\"\n", "references": [ "https://twitter.com/JohnLaTwC/status/1419251082736201737" @@ -40,5 +40,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_adfind_command_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_adfind_command_activity.json index 97ba7da6c5f3b..9af3832303666 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_adfind_command_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_adfind_command_activity.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "AdFind Command Activity", - "note": "## Triage and analysis\n\n`AdFind.exe` is a legitimate domain query tool. Rule alerts should be investigated to identify if the user has a role that would explain using this tool and that it is being run from an expected directory and endpoint. Leverage the exception workflow in the Kibana Security App or Elasticsearch API to tune this rule to your environment.", + "note": "## Triage and analysis\n\n### Investigating AdFind Command Activity\n\n[AdFind](http://www.joeware.net/freetools/tools/adfind/) is a freely available command-line tool used to retrieve information from\nActivity Directory (AD). Network discovery and enumeration tools like `AdFind` are useful to adversaries in the same ways\nthey are effective for network administrators. This tool provides quick ability to scope AD person/computer objects and\nunderstand subnets and domain information. There are many [examples](https://thedfirreport.com/category/adfind/)\nobserved where this tool has been adopted by ransomware and criminal groups and used in compromises.\n\n#### Possible investigation steps:\n- `AdFind` is a legitimate Active Directory enumeration tool used by network administrators, it's important to understand\nthe source of the activity. This could involve identifying the account using `AdFind` and determining based on the command-lines\nwhat information was retrieved, then further determining if these actions are in scope of that user's traditional responsibilities.\n- In multiple public references, `AdFind` is leveraged after initial access is achieved, review previous activity on impacted\nmachine looking for suspicious indicators such as previous anti-virus/EDR alerts, phishing emails received, or network traffic\nto suspicious infrastructure\n\n### False Positive Analysis\n- This rule has the high chance to produce false positives as it is a legitimate tool used by network administrators. One\noption could be whitelisting specific users or groups who use the tool as part of their daily responsibilities. This can\nbe done by leveraging the exception workflow in the Kibana Security App or Elasticsearch API to tune this rule to your environment\n- Malicious behavior with `AdFind` should be investigated as part of a step within an attack chain. It doesn't happen in\nisolation, so reviewing previous logs/activity from impacted machines could be very telling.\n\n### Related Rules\n- Windows Network Enumeration\n- Enumeration of Administrator Accounts\n- Enumeration Command Spawned via WMIPrvSE\n\n### Response and Remediation\n- Immediate response should be taken to validate activity, investigate and potentially isolate activity to prevent further\npost-compromise behavior\n- It's important to understand that `AdFind` is an Active Directory enumeration tool and can be used for malicious or legitimate\npurposes, so understanding the intent behind the activity will help determine the appropropriate response.\n", "query": "process where event.type in (\"start\", \"process_started\") and \n (process.name : \"AdFind.exe\" or process.pe.original_file_name == \"AdFind.exe\") and \n process.args : (\"objectcategory=computer\", \"(objectcategory=computer)\", \n \"objectcategory=person\", \"(objectcategory=person)\",\n \"objectcategory=subnet\", \"(objectcategory=subnet)\",\n \"objectcategory=group\", \"(objectcategory=group)\", \n \"objectcategory=organizationalunit\", \"(objectcategory=organizationalunit)\",\n \"objectcategory=attributeschema\", \"(objectcategory=attributeschema)\",\n \"domainlist\", \"dcmodes\", \"adinfo\", \"dclist\", \"computers_pwnotreqd\", \"trustdmp\")\n", "references": [ "http://www.joeware.net/freetools/tools/adfind/", @@ -69,11 +69,16 @@ "id": "T1482", "name": "Domain Trust Discovery", "reference": "https://attack.mitre.org/techniques/T1482/" + }, + { + "id": "T1018", + "name": "Remote System Discovery", + "reference": "https://attack.mitre.org/techniques/T1018/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_admin_recon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_admin_recon.json index 1a3ceebe7218f..d5026780fdf56 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_admin_recon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_admin_recon.json @@ -35,7 +35,14 @@ { "id": "T1069", "name": "Permission Groups Discovery", - "reference": "https://attack.mitre.org/techniques/T1069/" + "reference": "https://attack.mitre.org/techniques/T1069/", + "subtechnique": [ + { + "id": "T1069.002", + "name": "Domain Groups", + "reference": "https://attack.mitre.org/techniques/T1069/002/" + } + ] }, { "id": "T1087", @@ -47,5 +54,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json index 87b32d14791bb..dc855f3ed9a57 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json @@ -33,14 +33,14 @@ }, "technique": [ { - "id": "T1087", - "name": "Account Discovery", - "reference": "https://attack.mitre.org/techniques/T1087/" + "id": "T1033", + "name": "System Owner/User Discovery", + "reference": "https://attack.mitre.org/techniques/T1033/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 7 + "version": 8 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_security_software_wmic.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_security_software_wmic.json index d0f26c6e41756..92731ab40e78a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_security_software_wmic.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_security_software_wmic.json @@ -35,12 +35,19 @@ { "id": "T1518", "name": "Software Discovery", - "reference": "https://attack.mitre.org/techniques/T1518/" + "reference": "https://attack.mitre.org/techniques/T1518/", + "subtechnique": [ + { + "id": "T1518.001", + "name": "Security Software Discovery", + "reference": "https://attack.mitre.org/techniques/T1518/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting_grep.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting_grep.json new file mode 100644 index 0000000000000..e557e37db23d6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting_grep.json @@ -0,0 +1,52 @@ +{ + "author": [ + "Elastic" + ], + "description": "An adversary may attempt to get detailed information about the operating system and hardware. This rule identifies common locations used to discover virtual machine hardware by a non-root user. This technique has been used by the Pupy RAT and other malware.", + "false_positives": [ + "Certain tools or automated software may enumerate hardware information. These tools can be exempted via user name or process arguments to eliminate potential noise." + ], + "from": "now-9m", + "index": [ + "auditbeat-*", + "logs-endpoint.events.*" + ], + "language": "eql", + "license": "Elastic License v2", + "name": "Virtual Machine Fingerprinting via Grep", + "query": "process where event.type == \"start\" and\n process.name in (\"grep\", \"egrep\") and user.id != \"0\" and\n process.args : (\"parallels*\", \"vmware*\", \"virtualbox*\") and process.args : \"Manufacturer*\" and \n not process.parent.executable in (\"/Applications/Docker.app/Contents/MacOS/Docker\", \"/usr/libexec/kcare/virt-what\")\n", + "references": [ + "https://objective-see.com/blog/blog_0x4F.html" + ], + "risk_score": 47, + "rule_id": "c85eb82c-d2c8-485c-a36f-534f914b7663", + "severity": "medium", + "tags": [ + "Elastic", + "Host", + "macOS", + "Linux", + "Threat Detection", + "Discovery" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0007", + "name": "Discovery", + "reference": "https://attack.mitre.org/tactics/TA0007/" + }, + "technique": [ + { + "id": "T1082", + "name": "System Information Discovery", + "reference": "https://attack.mitre.org/techniques/T1082/" + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_enumeration_via_wmiprvse.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_enumeration_via_wmiprvse.json index 6a967d9644c47..441e01b4a1b12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_enumeration_via_wmiprvse.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_enumeration_via_wmiprvse.json @@ -38,9 +38,34 @@ "reference": "https://attack.mitre.org/techniques/T1047/" } ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0007", + "name": "Discovery", + "reference": "https://attack.mitre.org/tactics/TA0007/" + }, + "technique": [ + { + "id": "T1518", + "name": "Software Discovery", + "reference": "https://attack.mitre.org/techniques/T1518/" + }, + { + "id": "T1087", + "name": "Account Discovery", + "reference": "https://attack.mitre.org/techniques/T1087/" + }, + { + "id": "T1018", + "name": "Remote System Discovery", + "reference": "https://attack.mitre.org/techniques/T1018/" + } + ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json index abc41d9f6d5c3..094b87f33ada7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json @@ -34,7 +34,20 @@ "name": "Execution", "reference": "https://attack.mitre.org/tactics/TA0002/" }, - "technique": [] + "technique": [ + { + "id": "T1204", + "name": "User Execution", + "reference": "https://attack.mitre.org/techniques/T1204/", + "subtechnique": [ + { + "id": "T1204.002", + "name": "Malicious File", + "reference": "https://attack.mitre.org/techniques/T1204/002/" + } + ] + } + ] }, { "framework": "MITRE ATT&CK", @@ -60,5 +73,5 @@ } ], "type": "eql", - "version": 8 + "version": 9 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_scheduled_task_powershell_source.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_scheduled_task_powershell_source.json index 24492343e98c0..3814b00321417 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_scheduled_task_powershell_source.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_scheduled_task_powershell_source.json @@ -41,11 +41,18 @@ { "id": "T1053", "name": "Scheduled Task/Job", - "reference": "https://attack.mitre.org/techniques/T1053/" + "reference": "https://attack.mitre.org/techniques/T1053/", + "subtechnique": [ + { + "id": "T1053.005", + "name": "Scheduled Task", + "reference": "https://attack.mitre.org/techniques/T1053/005/" + } + ] } ] } ], "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json index efc3884b417fb..73c796c4e206d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json @@ -34,7 +34,20 @@ "name": "Execution", "reference": "https://attack.mitre.org/tactics/TA0002/" }, - "technique": [] + "technique": [ + { + "id": "T1204", + "name": "User Execution", + "reference": "https://attack.mitre.org/techniques/T1204/", + "subtechnique": [ + { + "id": "T1204.002", + "name": "Malicious File", + "reference": "https://attack.mitre.org/techniques/T1204/002/" + } + ] + } + ] }, { "framework": "MITRE ATT&CK", @@ -61,5 +74,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 8 + "version": 9 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_rds_snapshot_export.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_rds_snapshot_export.json index 430d97690b6f4..b59adc45b4236 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_rds_snapshot_export.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_rds_snapshot_export.json @@ -1,6 +1,7 @@ { "author": [ - "Elastic" + "Elastic", + "Austin Songer" ], "description": "Identifies the export of an Amazon Relational Database Service (RDS) Aurora database snapshot.", "false_positives": [ @@ -44,5 +45,5 @@ ], "timestamp_override": "event.ingested", "type": "query", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_backup_file_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_backup_file_deletion.json new file mode 100644 index 0000000000000..93c4c287d12ce --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_backup_file_deletion.json @@ -0,0 +1,52 @@ +{ + "author": [ + "Elastic" + ], + "description": "Identifies the deletion of backup files, saved using third-party software, by a process outside of the backup suite. Adversaries may delete Backup files to ensure that recovery from a Ransomware attack is less likely.", + "false_positives": [ + "Certain utilities that delete files for disk cleanup or Administrators manually removing backup files." + ], + "from": "now-9m", + "index": [ + "winlogbeat-*", + "logs-endpoint.events.*", + "logs-windows.*" + ], + "language": "eql", + "license": "Elastic License v2", + "name": "Third-party Backup Files Deleted via Unexpected Process", + "query": "file where event.type == \"deletion\" and\n (\n /* Veeam Related Backup Files */\n (file.extension : (\"VBK\", \"VIB\", \"VBM\") and\n not process.executable : (\"?:\\\\Windows\\\\Veeam\\\\Backup\\\\*\",\n \"?:\\\\Program Files\\\\Veeam\\\\Backup and Replication\\\\*\",\n \"?:\\\\Program Files (x86)\\\\Veeam\\\\Backup and Replication\\\\*\")) or\n\n /* Veritas Backup Exec Related Backup File */\n (file.extension : \"BKF\" and\n not process.executable : (\"?:\\\\Program Files\\\\Veritas\\\\Backup Exec\\\\*\",\n \"?:\\\\Program Files (x86)\\\\Veritas\\\\Backup Exec\\\\*\"))\n )\n", + "references": [ + "https://www.advintel.io/post/backup-removal-solutions-from-conti-ransomware-with-love" + ], + "risk_score": 47, + "rule_id": "11ea6bec-ebde-4d71-a8e9-784948f8e3e9", + "severity": "medium", + "tags": [ + "Elastic", + "Host", + "Windows", + "Threat Detection", + "Impact" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "Impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1490", + "name": "Inhibit System Recovery", + "reference": "https://attack.mitre.org/techniques/T1490/" + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_deleting_backup_catalogs_with_wbadmin.json similarity index 66% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_deleting_backup_catalogs_with_wbadmin.json index 5d1233ebfcb78..0c0c2a71b8263 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_deleting_backup_catalogs_with_wbadmin.json @@ -21,33 +21,26 @@ "Host", "Windows", "Threat Detection", - "Defense Evasion" + "Impact" ], "threat": [ { "framework": "MITRE ATT&CK", "tactic": { - "id": "TA0005", - "name": "Defense Evasion", - "reference": "https://attack.mitre.org/tactics/TA0005/" + "id": "TA0040", + "name": "Impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" }, "technique": [ { - "id": "T1070", - "name": "Indicator Removal on Host", - "reference": "https://attack.mitre.org/techniques/T1070/", - "subtechnique": [ - { - "id": "T1070.004", - "name": "File Deletion", - "reference": "https://attack.mitre.org/techniques/T1070/004/" - } - ] + "id": "T1490", + "name": "Inhibit System Recovery", + "reference": "https://attack.mitre.org/techniques/T1490/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 9 + "version": 10 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_microsoft_365_potential_ransomware_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_microsoft_365_potential_ransomware_activity.json new file mode 100644 index 0000000000000..52094400232b6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_microsoft_365_potential_ransomware_activity.json @@ -0,0 +1,54 @@ +{ + "author": [ + "Austin Songer" + ], + "description": "Identifies when Microsoft Cloud App Security reported when a user uploads files to the cloud that might be infected with ransomware.", + "false_positives": [ + "If Cloud App Security identifies, for example, a high rate of file uploads or file deletion activities it may represent an adverse encryption process." + ], + "from": "now-30m", + "index": [ + "filebeat-*", + "logs-o365*" + ], + "language": "kuery", + "license": "Elastic License v2", + "name": "Microsoft 365 Potential ransomware activity", + "note": "## Config\n\nThe Microsoft 365 Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n", + "query": "event.dataset:o365.audit and event.provider:SecurityComplianceCenter and event.category:web and event.action:\"Potential ransomware activity\" and event.outcome:success\n", + "references": [ + "https://docs.microsoft.com/en-us/cloud-app-security/anomaly-detection-policy", + "https://docs.microsoft.com/en-us/cloud-app-security/policy-template-reference" + ], + "risk_score": 47, + "rule_id": "721999d0-7ab2-44bf-b328-6e63367b9b29", + "severity": "medium", + "tags": [ + "Elastic", + "Cloud", + "Microsoft 365", + "Continuous Monitoring", + "SecOps", + "Configuration Audit" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "Impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1486", + "name": "Data Encrypted for Impact", + "reference": "https://attack.mitre.org/techniques/T1486/" + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_microsoft_365_unusual_volume_of_file_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_microsoft_365_unusual_volume_of_file_deletion.json new file mode 100644 index 0000000000000..c3a53310781df --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_microsoft_365_unusual_volume_of_file_deletion.json @@ -0,0 +1,54 @@ +{ + "author": [ + "Austin Songer" + ], + "description": "Identifies that a user has deleted an unusually large volume of files as reported by Microsoft Cloud App Security.", + "false_positives": [ + "Users or System Administrator cleaning out folders." + ], + "from": "now-30m", + "index": [ + "filebeat-*", + "logs-o365*" + ], + "language": "kuery", + "license": "Elastic License v2", + "name": "Microsoft 365 Unusual Volume of File Deletion", + "note": "## Config\n\nThe Microsoft 365 Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n", + "query": "event.dataset:o365.audit and event.provider:SecurityComplianceCenter and event.category:web and event.action:\"Unusual volume of file deletion\" and event.outcome:success\n", + "references": [ + "https://docs.microsoft.com/en-us/cloud-app-security/anomaly-detection-policy", + "https://docs.microsoft.com/en-us/cloud-app-security/policy-template-reference" + ], + "risk_score": 47, + "rule_id": "b2951150-658f-4a60-832f-a00d1e6c6745", + "severity": "medium", + "tags": [ + "Elastic", + "Cloud", + "Microsoft 365", + "Continuous Monitoring", + "SecOps", + "Configuration Audit" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "Impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1485", + "name": "Data Destruction", + "reference": "https://attack.mitre.org/techniques/T1485/" + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_modification_of_boot_config.json similarity index 69% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_modification_of_boot_config.json index 7c58d82ec1061..91f5959bee119 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_modification_of_boot_config.json @@ -21,33 +21,26 @@ "Host", "Windows", "Threat Detection", - "Defense Evasion" + "Impact" ], "threat": [ { "framework": "MITRE ATT&CK", "tactic": { - "id": "TA0005", - "name": "Defense Evasion", - "reference": "https://attack.mitre.org/tactics/TA0005/" + "id": "TA0040", + "name": "Impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" }, "technique": [ { - "id": "T1070", - "name": "Indicator Removal on Host", - "reference": "https://attack.mitre.org/techniques/T1070/", - "subtechnique": [ - { - "id": "T1070.004", - "name": "File Deletion", - "reference": "https://attack.mitre.org/techniques/T1070/004/" - } - ] + "id": "T1490", + "name": "Inhibit System Recovery", + "reference": "https://attack.mitre.org/techniques/T1490/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 8 + "version": 9 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_stop_process_service_threshold.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_stop_process_service_threshold.json similarity index 64% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_stop_process_service_threshold.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_stop_process_service_threshold.json index 86903058b62fe..ec361a8795538 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_stop_process_service_threshold.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_stop_process_service_threshold.json @@ -2,7 +2,7 @@ "author": [ "Elastic" ], - "description": "This rule identifies a high number (10) of process terminations (stop, delete, or suspend) from the same host within a short time period. This may indicate a defense evasion attempt.", + "description": "This rule identifies a high number (10) of process terminations (stop, delete, or suspend) from the same host within a short time period.", "from": "now-9m", "index": [ "winlogbeat-*", @@ -21,28 +21,21 @@ "Host", "Windows", "Threat Detection", - "Defense Evasion" + "Impact" ], "threat": [ { "framework": "MITRE ATT&CK", "tactic": { - "id": "TA0005", - "name": "Defense Evasion", - "reference": "https://attack.mitre.org/tactics/TA0005/" + "id": "TA0040", + "name": "Impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" }, "technique": [ { - "id": "T1562", - "name": "Impair Defenses", - "reference": "https://attack.mitre.org/techniques/T1562/", - "subtechnique": [ - { - "id": "T1562.001", - "name": "Disable or Modify Tools", - "reference": "https://attack.mitre.org/techniques/T1562/001/" - } - ] + "id": "T1489", + "name": "Service Stop", + "reference": "https://attack.mitre.org/techniques/T1489/" } ] } @@ -54,5 +47,5 @@ "value": 10 }, "type": "threshold", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_vssadmin.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_or_resized_via_vssadmin.json similarity index 70% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_vssadmin.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_or_resized_via_vssadmin.json index f0ac38e98441e..940229bf63751 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_vssadmin.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_or_resized_via_vssadmin.json @@ -2,7 +2,7 @@ "author": [ "Elastic" ], - "description": "Identifies use of vssadmin.exe for shadow copy deletion on endpoints. This commonly occurs in tandem with ransomware or other destructive attacks.", + "description": "Identifies use of vssadmin.exe for shadow copy deletion or resizing on endpoints. This commonly occurs in tandem with ransomware or other destructive attacks.", "from": "now-9m", "index": [ "winlogbeat-*", @@ -11,8 +11,8 @@ ], "language": "eql", "license": "Elastic License v2", - "name": "Volume Shadow Copy Deletion via VssAdmin", - "query": "process where event.type in (\"start\", \"process_started\") and\n (process.name : \"vssadmin.exe\" or process.pe.original_file_name == \"VSSADMIN.EXE\") and\n process.args : \"delete\" and process.args : \"shadows\"\n", + "name": "Volume Shadow Copy Deleted or Resized via VssAdmin", + "query": "process where event.type in (\"start\", \"process_started\") and event.action == \"start\" \n and (process.name : \"vssadmin.exe\" or process.pe.original_file_name == \"VSSADMIN.EXE\") and\n process.args in (\"delete\", \"resize\") and process.args : \"shadows*\"\n", "risk_score": 73, "rule_id": "b5ea4bfe-a1b2-421f-9d47-22a75a6f2921", "severity": "high", @@ -42,5 +42,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 9 + "version": 10 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_powershell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_powershell.json new file mode 100644 index 0000000000000..43dce4acf4df8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_powershell.json @@ -0,0 +1,52 @@ +{ + "author": [ + "Elastic", + "Austin Songer" + ], + "description": "Identifies the use of the Win32_ShadowCopy class and related cmdlets to achieve shadow copy deletion. This commonly occurs in tandem with ransomware or other destructive attacks.", + "from": "now-9m", + "index": [ + "winlogbeat-*", + "logs-endpoint.events.*", + "logs-windows.*" + ], + "language": "eql", + "license": "Elastic License v2", + "name": "Volume Shadow Copy Deletion via PowerShell", + "query": "process where event.type in (\"start\", \"process_started\") and\n process.name : (\"powershell.exe\", \"pwsh.exe\") and \n process.args : (\"*Get-WmiObject*\", \"*gwmi*\", \"*Get-CimInstance*\", \"*gcim*\") and\n process.args : (\"*Win32_ShadowCopy*\") and\n process.args : (\"*.Delete()*\", \"*Remove-WmiObject*\", \"*rwmi*\", \"*Remove-CimInstance*\", \"*rcim*\")\n", + "references": [ + "https://docs.microsoft.com/en-us/previous-versions/windows/desktop/vsswmi/win32-shadowcopy", + "https://powershell.one/wmi/root/cimv2/win32_shadowcopy", + "https://www.fortinet.com/blog/threat-research/stomping-shadow-copies-a-second-look-into-deletion-methods" + ], + "risk_score": 73, + "rule_id": "d99a037b-c8e2-47a5-97b9-170d076827c4", + "severity": "high", + "tags": [ + "Elastic", + "Host", + "Windows", + "Threat Detection", + "Impact" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "Impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1490", + "name": "Inhibit System Recovery", + "reference": "https://attack.mitre.org/techniques/T1490/" + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_wmic.json similarity index 66% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_wmic.json index e519b23a32b0d..f4f530362a5b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_volume_shadow_copy_deletion_via_wmic.json @@ -21,33 +21,26 @@ "Host", "Windows", "Threat Detection", - "Defense Evasion" + "Impact" ], "threat": [ { "framework": "MITRE ATT&CK", "tactic": { - "id": "TA0005", - "name": "Defense Evasion", - "reference": "https://attack.mitre.org/tactics/TA0005/" + "id": "TA0040", + "name": "Impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" }, "technique": [ { - "id": "T1070", - "name": "Indicator Removal on Host", - "reference": "https://attack.mitre.org/techniques/T1070/", - "subtechnique": [ - { - "id": "T1070.004", - "name": "File Deletion", - "reference": "https://attack.mitre.org/techniques/T1070/004/" - } - ] + "id": "T1490", + "name": "Inhibit System Recovery", + "reference": "https://attack.mitre.org/techniques/T1490/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 9 + "version": 10 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts index 093d5c806c282..1c5006f5e6f48 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts @@ -41,38 +41,38 @@ import rule28 from './command_and_control_vnc_virtual_network_computing_to_the_i import rule29 from './defense_evasion_adding_the_hidden_file_attribute_with_via_attribexe.json'; import rule30 from './defense_evasion_clearing_windows_event_logs.json'; import rule31 from './defense_evasion_delete_volume_usn_journal_with_fsutil.json'; -import rule32 from './defense_evasion_deleting_backup_catalogs_with_wbadmin.json'; -import rule33 from './defense_evasion_disable_windows_firewall_rules_with_netsh.json'; -import rule34 from './defense_evasion_misc_lolbin_connecting_to_the_internet.json'; -import rule35 from './defense_evasion_msbuild_making_network_connections.json'; -import rule36 from './defense_evasion_suspicious_certutil_commands.json'; -import rule37 from './defense_evasion_unusual_network_connection_via_rundll32.json'; -import rule38 from './defense_evasion_unusual_process_network_connection.json'; -import rule39 from './defense_evasion_via_filter_manager.json'; -import rule40 from './defense_evasion_volume_shadow_copy_deletion_via_wmic.json'; -import rule41 from './discovery_whoami_command_activity.json'; -import rule42 from './endgame_adversary_behavior_detected.json'; -import rule43 from './endgame_cred_dumping_detected.json'; -import rule44 from './endgame_cred_dumping_prevented.json'; -import rule45 from './endgame_cred_manipulation_detected.json'; -import rule46 from './endgame_cred_manipulation_prevented.json'; -import rule47 from './endgame_exploit_detected.json'; -import rule48 from './endgame_exploit_prevented.json'; -import rule49 from './endgame_malware_detected.json'; -import rule50 from './endgame_malware_prevented.json'; -import rule51 from './endgame_permission_theft_detected.json'; -import rule52 from './endgame_permission_theft_prevented.json'; -import rule53 from './endgame_process_injection_detected.json'; -import rule54 from './endgame_process_injection_prevented.json'; -import rule55 from './endgame_ransomware_detected.json'; -import rule56 from './endgame_ransomware_prevented.json'; -import rule57 from './execution_command_prompt_connecting_to_the_internet.json'; -import rule58 from './execution_command_shell_started_by_svchost.json'; -import rule59 from './execution_html_help_executable_program_connecting_to_the_internet.json'; -import rule60 from './execution_psexec_lateral_movement_command.json'; -import rule61 from './execution_register_server_program_connecting_to_the_internet.json'; -import rule62 from './execution_via_compiled_html_file.json'; -import rule63 from './impact_volume_shadow_copy_deletion_via_vssadmin.json'; +import rule32 from './defense_evasion_disable_windows_firewall_rules_with_netsh.json'; +import rule33 from './defense_evasion_misc_lolbin_connecting_to_the_internet.json'; +import rule34 from './defense_evasion_msbuild_making_network_connections.json'; +import rule35 from './defense_evasion_suspicious_certutil_commands.json'; +import rule36 from './defense_evasion_unusual_network_connection_via_rundll32.json'; +import rule37 from './defense_evasion_unusual_process_network_connection.json'; +import rule38 from './defense_evasion_via_filter_manager.json'; +import rule39 from './discovery_whoami_command_activity.json'; +import rule40 from './endgame_adversary_behavior_detected.json'; +import rule41 from './endgame_cred_dumping_detected.json'; +import rule42 from './endgame_cred_dumping_prevented.json'; +import rule43 from './endgame_cred_manipulation_detected.json'; +import rule44 from './endgame_cred_manipulation_prevented.json'; +import rule45 from './endgame_exploit_detected.json'; +import rule46 from './endgame_exploit_prevented.json'; +import rule47 from './endgame_malware_detected.json'; +import rule48 from './endgame_malware_prevented.json'; +import rule49 from './endgame_permission_theft_detected.json'; +import rule50 from './endgame_permission_theft_prevented.json'; +import rule51 from './endgame_process_injection_detected.json'; +import rule52 from './endgame_process_injection_prevented.json'; +import rule53 from './endgame_ransomware_detected.json'; +import rule54 from './endgame_ransomware_prevented.json'; +import rule55 from './execution_command_prompt_connecting_to_the_internet.json'; +import rule56 from './execution_command_shell_started_by_svchost.json'; +import rule57 from './execution_html_help_executable_program_connecting_to_the_internet.json'; +import rule58 from './execution_psexec_lateral_movement_command.json'; +import rule59 from './execution_register_server_program_connecting_to_the_internet.json'; +import rule60 from './execution_via_compiled_html_file.json'; +import rule61 from './impact_deleting_backup_catalogs_with_wbadmin.json'; +import rule62 from './impact_volume_shadow_copy_deletion_or_resized_via_vssadmin.json'; +import rule63 from './impact_volume_shadow_copy_deletion_via_wmic.json'; import rule64 from './initial_access_rpc_remote_procedure_call_from_the_internet.json'; import rule65 from './initial_access_rpc_remote_procedure_call_to_the_internet.json'; import rule66 from './initial_access_script_executing_powershell.json'; @@ -95,7 +95,7 @@ import rule82 from './persistence_system_shells_via_services.json'; import rule83 from './persistence_user_account_creation.json'; import rule84 from './persistence_via_application_shimming.json'; import rule85 from './privilege_escalation_unusual_parentchild_relationship.json'; -import rule86 from './defense_evasion_modification_of_boot_config.json'; +import rule86 from './impact_modification_of_boot_config.json'; import rule87 from './privilege_escalation_uac_bypass_event_viewer.json'; import rule88 from './defense_evasion_msxsl_network.json'; import rule89 from './discovery_net_command_system_account.json'; @@ -328,7 +328,7 @@ import rule315 from './command_and_control_cobalt_strike_default_teamserver_cert import rule316 from './defense_evasion_enable_inbound_rdp_with_netsh.json'; import rule317 from './defense_evasion_execution_lolbas_wuauclt.json'; import rule318 from './privilege_escalation_unusual_svchost_childproc_childless.json'; -import rule319 from './lateral_movement_rdp_tunnel_plink.json'; +import rule319 from './command_and_control_rdp_tunnel_plink.json'; import rule320 from './privilege_escalation_uac_bypass_winfw_mmc_hijack.json'; import rule321 from './persistence_ms_office_addins_file.json'; import rule322 from './discovery_adfind_command_activity.json'; @@ -428,8 +428,8 @@ import rule415 from './credential_access_copy_ntds_sam_volshadowcp_cmdline.json' import rule416 from './credential_access_lsass_memdump_file_created.json'; import rule417 from './lateral_movement_incoming_winrm_shell_execution.json'; import rule418 from './lateral_movement_powershell_remoting_target.json'; -import rule419 from './defense_evasion_hide_encoded_executable_registry.json'; -import rule420 from './defense_evasion_port_forwarding_added_registry.json'; +import rule419 from './command_and_control_port_forwarding_added_registry.json'; +import rule420 from './defense_evasion_hide_encoded_executable_registry.json'; import rule421 from './lateral_movement_rdp_enabled_registry.json'; import rule422 from './privilege_escalation_printspooler_registry_copyfiles.json'; import rule423 from './privilege_escalation_rogue_windir_environment_var.json'; @@ -443,7 +443,7 @@ import rule430 from './credential_access_microsoft_365_brute_force_user_account_ import rule431 from './microsoft_365_teams_custom_app_interaction_allowed.json'; import rule432 from './persistence_microsoft_365_teams_external_access_enabled.json'; import rule433 from './credential_access_microsoft_365_potential_password_spraying_attack.json'; -import rule434 from './defense_evasion_stop_process_service_threshold.json'; +import rule434 from './impact_stop_process_service_threshold.json'; import rule435 from './collection_winrar_encryption.json'; import rule436 from './defense_evasion_unusual_dir_ads.json'; import rule437 from './discovery_admin_recon.json'; @@ -466,8 +466,8 @@ import rule453 from './execution_apt_solarwinds_backdoor_child_cmd_powershell.js import rule454 from './execution_apt_solarwinds_backdoor_unusual_child_processes.json'; import rule455 from './initial_access_azure_active_directory_powershell_signin.json'; import rule456 from './collection_email_powershell_exchange_mailbox.json'; -import rule457 from './collection_persistence_powershell_exch_mailbox_activesync_add_device.json'; -import rule458 from './execution_scheduled_task_powershell_source.json'; +import rule457 from './execution_scheduled_task_powershell_source.json'; +import rule458 from './persistence_powershell_exch_mailbox_activesync_add_device.json'; import rule459 from './persistence_docker_shortcuts_plist_modification.json'; import rule460 from './persistence_evasion_hidden_local_account_creation.json'; import rule461 from './persistence_finder_sync_plugin_pluginkit.json'; @@ -551,36 +551,54 @@ import rule538 from './persistence_ec2_security_group_configuration_change_detec import rule539 from './defense_evasion_disabling_windows_logs.json'; import rule540 from './persistence_route_53_domain_transfer_lock_disabled.json'; import rule541 from './persistence_route_53_domain_transferred_to_another_account.json'; -import rule542 from './credential_access_user_excessive_sso_logon_errors.json'; -import rule543 from './defense_evasion_suspicious_execution_from_mounted_device.json'; -import rule544 from './defense_evasion_unusual_network_connection_via_dllhost.json'; -import rule545 from './defense_evasion_amsienable_key_mod.json'; -import rule546 from './impact_rds_group_deletion.json'; -import rule547 from './persistence_rds_group_creation.json'; -import rule548 from './exfiltration_rds_snapshot_export.json'; -import rule549 from './persistence_rds_instance_creation.json'; -import rule550 from './ml_auth_rare_hour_for_a_user_to_logon.json'; -import rule551 from './ml_auth_rare_source_ip_for_a_user.json'; -import rule552 from './ml_auth_rare_user_logon.json'; -import rule553 from './ml_auth_spike_in_failed_logon_events.json'; -import rule554 from './ml_auth_spike_in_logon_events.json'; -import rule555 from './ml_auth_spike_in_logon_events_from_a_source_ip.json'; -import rule556 from './privilege_escalation_cyberarkpas_error_audit_event_promotion.json'; -import rule557 from './privilege_escalation_cyberarkpas_recommended_events_to_monitor_promotion.json'; -import rule558 from './privilege_escalation_printspooler_malicious_driver_file_changes.json'; -import rule559 from './privilege_escalation_printspooler_malicious_registry_modification.json'; -import rule560 from './privilege_escalation_printspooler_suspicious_file_deletion.json'; -import rule561 from './privilege_escalation_unusual_printspooler_childprocess.json'; -import rule562 from './defense_evasion_disabling_windows_defender_powershell.json'; -import rule563 from './defense_evasion_enable_network_discovery_with_netsh.json'; -import rule564 from './defense_evasion_execution_windefend_unusual_path.json'; -import rule565 from './defense_evasion_agent_spoofing_mismatched_id.json'; -import rule566 from './defense_evasion_agent_spoofing_multiple_hosts.json'; -import rule567 from './defense_evasion_parent_process_pid_spoofing.json'; -import rule568 from './defense_evasion_defender_exclusion_via_powershell.json'; -import rule569 from './defense_evasion_whitespace_padding_in_command_line.json'; -import rule570 from './persistence_webshell_detection.json'; -import rule571 from './persistence_via_bits_job_notify_command.json'; +import rule542 from './initial_access_okta_user_attempted_unauthorized_access.json'; +import rule543 from './credential_access_user_excessive_sso_logon_errors.json'; +import rule544 from './persistence_exchange_suspicious_mailbox_right_delegation.json'; +import rule545 from './privilege_escalation_new_or_modified_federation_domain.json'; +import rule546 from './privilege_escalation_sts_getsessiontoken_abuse.json'; +import rule547 from './defense_evasion_suspicious_execution_from_mounted_device.json'; +import rule548 from './defense_evasion_unusual_network_connection_via_dllhost.json'; +import rule549 from './defense_evasion_amsienable_key_mod.json'; +import rule550 from './impact_rds_group_deletion.json'; +import rule551 from './persistence_rds_group_creation.json'; +import rule552 from './persistence_route_table_modified_or_deleted.json'; +import rule553 from './exfiltration_rds_snapshot_export.json'; +import rule554 from './persistence_rds_instance_creation.json'; +import rule555 from './ml_auth_rare_hour_for_a_user_to_logon.json'; +import rule556 from './ml_auth_rare_source_ip_for_a_user.json'; +import rule557 from './ml_auth_rare_user_logon.json'; +import rule558 from './ml_auth_spike_in_failed_logon_events.json'; +import rule559 from './ml_auth_spike_in_logon_events.json'; +import rule560 from './ml_auth_spike_in_logon_events_from_a_source_ip.json'; +import rule561 from './privilege_escalation_cyberarkpas_error_audit_event_promotion.json'; +import rule562 from './privilege_escalation_cyberarkpas_recommended_events_to_monitor_promotion.json'; +import rule563 from './privilege_escalation_printspooler_malicious_driver_file_changes.json'; +import rule564 from './privilege_escalation_printspooler_malicious_registry_modification.json'; +import rule565 from './privilege_escalation_printspooler_suspicious_file_deletion.json'; +import rule566 from './privilege_escalation_unusual_printspooler_childprocess.json'; +import rule567 from './defense_evasion_disabling_windows_defender_powershell.json'; +import rule568 from './defense_evasion_enable_network_discovery_with_netsh.json'; +import rule569 from './defense_evasion_execution_windefend_unusual_path.json'; +import rule570 from './defense_evasion_agent_spoofing_mismatched_id.json'; +import rule571 from './defense_evasion_agent_spoofing_multiple_hosts.json'; +import rule572 from './defense_evasion_parent_process_pid_spoofing.json'; +import rule573 from './impact_microsoft_365_potential_ransomware_activity.json'; +import rule574 from './impact_microsoft_365_unusual_volume_of_file_deletion.json'; +import rule575 from './initial_access_microsoft_365_user_restricted_from_sending_email.json'; +import rule576 from './defense_evasion_elasticache_security_group_creation.json'; +import rule577 from './defense_evasion_elasticache_security_group_modified_or_deleted.json'; +import rule578 from './impact_volume_shadow_copy_deletion_via_powershell.json'; +import rule579 from './defense_evasion_defender_exclusion_via_powershell.json'; +import rule580 from './defense_evasion_whitespace_padding_in_command_line.json'; +import rule581 from './defense_evasion_frontdoor_firewall_policy_deletion.json'; +import rule582 from './persistence_webshell_detection.json'; +import rule583 from './defense_evasion_execution_control_panel_suspicious_args.json'; +import rule584 from './credential_access_potential_lsa_memdump_via_mirrordump.json'; +import rule585 from './discovery_virtual_machine_fingerprinting_grep.json'; +import rule586 from './impact_backup_file_deletion.json'; +import rule587 from './persistence_screensaver_engine_unexpected_child_process.json'; +import rule588 from './persistence_screensaver_plist_file_modification.json'; +import rule589 from './persistence_via_bits_job_notify_command.json'; export const rawRules = [ rule1, @@ -1154,4 +1172,22 @@ export const rawRules = [ rule569, rule570, rule571, + rule572, + rule573, + rule574, + rule575, + rule576, + rule577, + rule578, + rule579, + rule580, + rule581, + rule582, + rule583, + rule584, + rule585, + rule586, + rule587, + rule588, + rule589, ]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_microsoft_365_user_restricted_from_sending_email.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_microsoft_365_user_restricted_from_sending_email.json new file mode 100644 index 0000000000000..31950fc345c0e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_microsoft_365_user_restricted_from_sending_email.json @@ -0,0 +1,54 @@ +{ + "author": [ + "Austin Songer" + ], + "description": "Identifies when a user has been restricted from sending email due to exceeding sending limits of the service policies per the Security Compliance Center.", + "false_positives": [ + "A user sending emails using personal distribution folders may trigger the event." + ], + "from": "now-30m", + "index": [ + "filebeat-*", + "logs-o365*" + ], + "language": "kuery", + "license": "Elastic License v2", + "name": "Microsoft 365 User Restricted from Sending Email", + "note": "## Config\n\nThe Microsoft 365 Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n", + "query": "event.dataset:o365.audit and event.provider:SecurityComplianceCenter and event.category:web and event.action:\"User restricted from sending email\" and event.outcome:success\n", + "references": [ + "https://docs.microsoft.com/en-us/cloud-app-security/anomaly-detection-policy", + "https://docs.microsoft.com/en-us/cloud-app-security/policy-template-reference" + ], + "risk_score": 47, + "rule_id": "0136b315-b566-482f-866c-1d8e2477ba16", + "severity": "medium", + "tags": [ + "Elastic", + "Cloud", + "Microsoft 365", + "Continuous Monitoring", + "SecOps", + "Configuration Audit" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0001", + "name": "Initial Access", + "reference": "https://attack.mitre.org/tactics/TA0001/" + }, + "technique": [ + { + "id": "T1078", + "name": "Valid Accounts", + "reference": "https://attack.mitre.org/techniques/T1078/" + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_okta_user_attempted_unauthorized_access.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_okta_user_attempted_unauthorized_access.json new file mode 100644 index 0000000000000..222d30723bc9e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_okta_user_attempted_unauthorized_access.json @@ -0,0 +1,74 @@ +{ + "author": [ + "Elastic", + "Austin Songer" + ], + "description": "Identifies when an unauthorized access attempt is made by a user for an Okta application.", + "index": [ + "filebeat-*", + "logs-okta*" + ], + "language": "kuery", + "license": "Elastic License v2", + "name": "Unauthorized Access to an Okta Application", + "note": "## Config\n\nThe Okta Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.", + "query": "event.dataset:okta.system and event.action:app.generic.unauth_app_access_attempt\n", + "risk_score": 21, + "rule_id": "4edd3e1a-3aa0-499b-8147-4d2ea43b1613", + "severity": "low", + "tags": [ + "Elastic", + "Identity", + "Okta", + "Continuous Monitoring", + "SecOps", + "Identity and Access" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0001", + "name": "Initial Access", + "reference": "https://attack.mitre.org/tactics/TA0001/" + }, + "technique": [ + { + "id": "T1078", + "name": "Valid Accounts", + "reference": "https://attack.mitre.org/techniques/T1078/" + } + ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0005", + "name": "Defense Evasion", + "reference": "https://attack.mitre.org/tactics/TA0005/" + }, + "technique": [] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0003", + "name": "Persistence", + "reference": "https://attack.mitre.org/tactics/TA0003/" + }, + "technique": [] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0004", + "name": "Privilege Escalation", + "reference": "https://attack.mitre.org/tactics/TA0004/" + }, + "technique": [] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_ms_office_child_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_ms_office_child_process.json index 16486590cb093..17e9195181f3d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_ms_office_child_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_ms_office_child_process.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Suspicious MS Office Child Process", - "query": "process where event.type in (\"start\", \"process_started\") and\n process.parent.name : (\"eqnedt32.exe\", \"excel.exe\", \"fltldr.exe\", \"msaccess.exe\", \"mspub.exe\", \"powerpnt.exe\", \"winword.exe\") and\n process.name : (\"Microsoft.Workflow.Compiler.exe\", \"arp.exe\", \"atbroker.exe\", \"bginfo.exe\", \"bitsadmin.exe\", \"cdb.exe\", \"certutil.exe\",\n \"cmd.exe\", \"cmstp.exe\", \"cscript.exe\", \"csi.exe\", \"dnx.exe\", \"dsget.exe\", \"dsquery.exe\", \"forfiles.exe\", \"fsi.exe\",\n \"ftp.exe\", \"gpresult.exe\", \"hostname.exe\", \"ieexec.exe\", \"iexpress.exe\", \"installutil.exe\", \"ipconfig.exe\", \"mshta.exe\",\n \"msxsl.exe\", \"nbtstat.exe\", \"net.exe\", \"net1.exe\", \"netsh.exe\", \"netstat.exe\", \"nltest.exe\", \"odbcconf.exe\", \"ping.exe\",\n \"powershell.exe\", \"pwsh.exe\", \"qprocess.exe\", \"quser.exe\", \"qwinsta.exe\", \"rcsi.exe\", \"reg.exe\", \"regasm.exe\", \"regsvcs.exe\",\n \"regsvr32.exe\", \"sc.exe\", \"schtasks.exe\", \"systeminfo.exe\", \"tasklist.exe\", \"tracert.exe\", \"whoami.exe\",\n \"wmic.exe\", \"wscript.exe\", \"xwizard.exe\", \"explorer.exe\", \"rundll32.exe\", \"hh.exe\")\n", + "query": "process where event.type in (\"start\", \"process_started\") and\n process.parent.name : (\"eqnedt32.exe\", \"excel.exe\", \"fltldr.exe\", \"msaccess.exe\", \"mspub.exe\", \"powerpnt.exe\", \"winword.exe\") and\n process.name : (\"Microsoft.Workflow.Compiler.exe\", \"arp.exe\", \"atbroker.exe\", \"bginfo.exe\", \"bitsadmin.exe\", \"cdb.exe\", \"certutil.exe\",\n \"cmd.exe\", \"cmstp.exe\", \"control.exe\", \"cscript.exe\", \"csi.exe\", \"dnx.exe\", \"dsget.exe\", \"dsquery.exe\", \"forfiles.exe\", \n \"fsi.exe\", \"ftp.exe\", \"gpresult.exe\", \"hostname.exe\", \"ieexec.exe\", \"iexpress.exe\", \"installutil.exe\", \"ipconfig.exe\", \n \"mshta.exe\", \"msxsl.exe\", \"nbtstat.exe\", \"net.exe\", \"net1.exe\", \"netsh.exe\", \"netstat.exe\", \"nltest.exe\", \"odbcconf.exe\", \n \"ping.exe\", \"powershell.exe\", \"pwsh.exe\", \"qprocess.exe\", \"quser.exe\", \"qwinsta.exe\", \"rcsi.exe\", \"reg.exe\", \"regasm.exe\", \n \"regsvcs.exe\", \"regsvr32.exe\", \"sc.exe\", \"schtasks.exe\", \"systeminfo.exe\", \"tasklist.exe\", \"tracert.exe\", \"whoami.exe\",\n \"wmic.exe\", \"wscript.exe\", \"xwizard.exe\", \"explorer.exe\", \"rundll32.exe\", \"hh.exe\")\n", "risk_score": 47, "rule_id": "a624863f-a70d-417f-a7d2-7a404638d47f", "severity": "medium", @@ -49,5 +49,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 8 + "version": 9 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_credential_access_kerberos_bifrostconsole.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_credential_access_kerberos_bifrostconsole.json index 82fa9d8d72a92..0fd10fc807846 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_credential_access_kerberos_bifrostconsole.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_credential_access_kerberos_bifrostconsole.json @@ -60,12 +60,19 @@ { "id": "T1558", "name": "Steal or Forge Kerberos Tickets", - "reference": "https://attack.mitre.org/techniques/T1558/" + "reference": "https://attack.mitre.org/techniques/T1558/", + "subtechnique": [ + { + "id": "T1558.003", + "name": "Kerberoasting", + "reference": "https://attack.mitre.org/techniques/T1558/003/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "query", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_hta.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_hta.json index 707596aa333d0..f832eb51336f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_hta.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_hta.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Incoming DCOM Lateral Movement via MSHTA", - "query": "sequence with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and\n process.name : \"mshta.exe\" and process.args : \"-Embedding\"\n ] by host.id, process.entity_id\n [network where event.type == \"start\" and process.name : \"mshta.exe\" and \n network.direction == \"incoming\" and network.transport == \"tcp\" and\n source.port > 49151 and destination.port > 49151 and not source.address in (\"127.0.0.1\", \"::1\")\n ] by host.id, process.entity_id\n", + "query": "sequence with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and\n process.name : \"mshta.exe\" and process.args : \"-Embedding\"\n ] by host.id, process.entity_id\n [network where event.type == \"start\" and process.name : \"mshta.exe\" and \n network.direction : (\"incoming\", \"ingress\") and network.transport == \"tcp\" and\n source.port > 49151 and destination.port > 49151 and not source.address in (\"127.0.0.1\", \"::1\")\n ] by host.id, process.entity_id\n", "references": [ "https://codewhitesec.blogspot.com/2018/07/lethalhta.html" ], @@ -38,11 +38,40 @@ { "id": "T1021", "name": "Remote Services", - "reference": "https://attack.mitre.org/techniques/T1021/" + "reference": "https://attack.mitre.org/techniques/T1021/", + "subtechnique": [ + { + "id": "T1021.003", + "name": "Distributed Component Object Model", + "reference": "https://attack.mitre.org/techniques/T1021/003/" + } + ] + } + ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0005", + "name": "Defense Evasion", + "reference": "https://attack.mitre.org/tactics/TA0005/" + }, + "technique": [ + { + "id": "T1218", + "name": "Signed Binary Proxy Execution", + "reference": "https://attack.mitre.org/techniques/T1218/", + "subtechnique": [ + { + "id": "T1218.005", + "name": "Mshta", + "reference": "https://attack.mitre.org/techniques/T1218/005/" + } + ] } ] } ], "type": "eql", - "version": 2 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_mmc20.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_mmc20.json index c78343223a10f..8cb2e2c3690e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_mmc20.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_mmc20.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Incoming DCOM Lateral Movement with MMC", - "query": "sequence by host.id with maxspan=1m\n [network where event.type == \"start\" and process.name : \"mmc.exe\" and\n source.port >= 49152 and destination.port >= 49152 and source.address not in (\"127.0.0.1\", \"::1\") and\n network.direction == \"incoming\" and network.transport == \"tcp\"\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.parent.name : \"mmc.exe\"\n ] by process.parent.entity_id\n", + "query": "sequence by host.id with maxspan=1m\n [network where event.type == \"start\" and process.name : \"mmc.exe\" and\n source.port >= 49152 and destination.port >= 49152 and source.address not in (\"127.0.0.1\", \"::1\") and\n network.direction : (\"incoming\", \"ingress\") and network.transport == \"tcp\"\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.parent.name : \"mmc.exe\"\n ] by process.parent.entity_id\n", "references": [ "https://enigma0x3.net/2017/01/05/lateral-movement-using-the-mmc20-application-com-object/" ], @@ -38,11 +38,18 @@ { "id": "T1021", "name": "Remote Services", - "reference": "https://attack.mitre.org/techniques/T1021/" + "reference": "https://attack.mitre.org/techniques/T1021/", + "subtechnique": [ + { + "id": "T1021.003", + "name": "Distributed Component Object Model", + "reference": "https://attack.mitre.org/techniques/T1021/003/" + } + ] } ] } ], "type": "eql", - "version": 2 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_shellwindow_shellbrowserwindow.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_shellwindow_shellbrowserwindow.json index 617cbc2fab05e..9ca759cc2facd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_shellwindow_shellbrowserwindow.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dcom_shellwindow_shellbrowserwindow.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Incoming DCOM Lateral Movement with ShellBrowserWindow or ShellWindows", - "query": "sequence by host.id with maxspan=5s\n [network where event.type == \"start\" and process.name : \"explorer.exe\" and\n network.direction == \"incoming\" and network.transport == \"tcp\" and\n source.port > 49151 and destination.port > 49151 and not source.address in (\"127.0.0.1\", \"::1\")\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n process.parent.name : \"explorer.exe\"\n ] by process.parent.entity_id\n", + "query": "sequence by host.id with maxspan=5s\n [network where event.type == \"start\" and process.name : \"explorer.exe\" and\n network.direction : (\"incoming\", \"ingress\") and network.transport == \"tcp\" and\n source.port > 49151 and destination.port > 49151 and not source.address in (\"127.0.0.1\", \"::1\")\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n process.parent.name : \"explorer.exe\"\n ] by process.parent.entity_id\n", "references": [ "https://enigma0x3.net/2017/01/23/lateral-movement-via-dcom-round-2/" ], @@ -38,11 +38,18 @@ { "id": "T1021", "name": "Remote Services", - "reference": "https://attack.mitre.org/techniques/T1021/" + "reference": "https://attack.mitre.org/techniques/T1021/", + "subtechnique": [ + { + "id": "T1021.003", + "name": "Distributed Component Object Model", + "reference": "https://attack.mitre.org/techniques/T1021/003/" + } + ] } ] } ], "type": "eql", - "version": 2 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json index b4534c48d0fa2..c9983d2ba186e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json @@ -33,13 +33,20 @@ }, "technique": [ { - "id": "T1210", - "name": "Exploitation of Remote Services", - "reference": "https://attack.mitre.org/techniques/T1210/" + "id": "T1021", + "name": "Remote Services", + "reference": "https://attack.mitre.org/techniques/T1021/", + "subtechnique": [ + { + "id": "T1021.002", + "name": "SMB/Windows Admin Shares", + "reference": "https://attack.mitre.org/techniques/T1021/002/" + } + ] } ] } ], "type": "eql", - "version": 6 + "version": 7 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json index b34badc7c8611..6e11258e23d00 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json @@ -13,7 +13,7 @@ "language": "kuery", "license": "Elastic License v2", "name": "Abnormally Large DNS Response", - "note": "## Triage and analysis\n\n### Investigating Large DNS Responses\nDetection alerts from this rule indicate an attempt was made to exploit CVE-2020-1350 (SigRed) through the use of large DNS responses on a Windows DNS server. Here are some possible avenues of investigation:\n- Investigate any corresponding Intrusion Detection Signatures (IDS) alerts that can validate this detection alert.\n- Examine the `dns.question_type` network fieldset with a protocol analyzer, such as Zeek, Packetbeat, or Suricata, for `SIG` or `RRSIG` data.\n- Validate the patch level and OS of the targeted DNS server to validate the observed activity was not large-scale Internet vulnerability scanning.\n- Validate that the source of the network activity was not from an authorized vulnerability scan or compromise assessment.", + "note": "## Triage and analysis\n\n### Investigating Large DNS Responses\nDetection alerts from this rule indicate possible anomalous activity around large byte DNS responses from a Windows DNS\nserver. This detection rule was created based on activity represented in exploitation of vulnerability (CVE-2020-1350)\nalso known as [SigRed](https://www.elastic.co/blog/detection-rules-for-sigred-vulnerability) during July 2020.\n\n#### Possible investigation steps:\n- This specific rule is sourced from network log activity such as DNS or network level data. It's important to validate\nthe source of the incoming traffic and determine if this activity has been observed previously within an environment.\n- Activity can be further investigated and validated by reviewing available corresponding Intrusion Detection Signatures (IDS) alerts associated with activity.\n- Further examination can be made by reviewing the `dns.question_type` network fieldset with a protocol analyzer, such as Zeek, Packetbeat, or Suricata, for `SIG` or `RRSIG` data.\n- Validate the patch level and OS of the targeted DNS server to validate the observed activity was not large-scale Internet vulnerability scanning.\n- Validate that the source of the network activity was not from an authorized vulnerability scan or compromise assessment.\n\n#### False Positive Analysis\n- Based on this rule which looks for a threshold of 60k bytes, it is possible for activity to be generated under 65k bytes\nand related to legitimate behavior. In packet capture files received by the [SANS Internet Storm Center](https://isc.sans.edu/forums/diary/PATCH+NOW+SIGRed+CVE20201350+Microsoft+DNS+Server+Vulnerability/26356/), byte responses\nwere all observed as greater than 65k bytes.\n- This activity has the ability to be triggered from compliance/vulnerability scanning or compromise assessment, it's\nimportant to determine the source of the activity and potential whitelist the source host\n\n\n### Related Rules\n- Unusual Child Process of dns.exe\n- Unusual File Modification by dns.exe\n\n### Response and Remediation\n- Review and implement the above detection logic within your environment using technology such as Endpoint security, Winlogbeat, Packetbeat, or network security monitoring (NSM) platforms such as Zeek or Suricata.\n- Ensure that you have deployed the latest Microsoft [Security Update](https://portal.msrc.microsoft.com/en-US/security-guidance/advisory/CVE-2020-1350) (Monthly Rollup or Security Only) and restart the\npatched machines. If unable to patch immediately: Microsoft [released](https://support.microsoft.com/en-us/help/4569509/windows-dns-server-remote-code-execution-vulnerability) a registry-based workaround that doesn\u2019t require a\nrestart. This can be used as a temporary solution before the patch is applied.\n- Maintain backups of your critical systems to aid in quick recovery.\n- Perform routine vulnerability scans of your systems, monitor [CISA advisories](https://us-cert.cisa.gov/ncas/current-activity) and patch identified vulnerabilities.\n- If observed true positive activity, implement a remediation plan and monitor host-based artifacts for additional post-exploitation behavior.\n", "query": "event.category:(network or network_traffic) and destination.port:53 and\n (event.dataset:zeek.dns or type:dns or event.type:connection) and network.bytes > 60000\n", "references": [ "https://research.checkpoint.com/2020/resolving-your-way-into-domain-admin-exploiting-a-17-year-old-bug-in-windows-dns-servers/", @@ -48,5 +48,5 @@ ], "timestamp_override": "event.ingested", "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_executable_tool_transfer_smb.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_executable_tool_transfer_smb.json index 8173ddc6f1003..5fe9d066bc76d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_executable_tool_transfer_smb.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_executable_tool_transfer_smb.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Lateral Tool Transfer", - "query": "sequence by host.id with maxspan=30s\n [network where event.type == \"start\" and process.pid == 4 and destination.port == 445 and\n network.direction == \"incoming\" and network.transport == \"tcp\" and\n source.address != \"127.0.0.1\" and source.address != \"::1\"\n ] by process.entity_id\n /* add more executable extensions here if they are not noisy in your environment */\n [file where event.type in (\"creation\", \"change\") and process.pid == 4 and file.extension : (\"exe\", \"dll\", \"bat\", \"cmd\")] by process.entity_id\n", + "query": "sequence by host.id with maxspan=30s\n [network where event.type == \"start\" and process.pid == 4 and destination.port == 445 and\n network.direction : (\"incoming\", \"ingress\") and network.transport == \"tcp\" and\n source.address != \"127.0.0.1\" and source.address != \"::1\"\n ] by process.entity_id\n /* add more executable extensions here if they are not noisy in your environment */\n [file where event.type in (\"creation\", \"change\") and process.pid == 4 and file.extension : (\"exe\", \"dll\", \"bat\", \"cmd\")] by process.entity_id\n", "risk_score": 47, "rule_id": "58bc134c-e8d2-4291-a552-b4b3e537c60b", "severity": "medium", @@ -41,5 +41,5 @@ } ], "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_incoming_winrm_shell_execution.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_incoming_winrm_shell_execution.json index 062013549e1da..04a60f99556f4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_incoming_winrm_shell_execution.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_incoming_winrm_shell_execution.json @@ -15,7 +15,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Incoming Execution via WinRM Remote Shell", - "query": "sequence by host.id with maxspan=30s\n [network where process.pid == 4 and network.direction == \"incoming\" and\n destination.port in (5985, 5986) and network.protocol == \"http\" and not source.address in (\"::1\", \"127.0.0.1\")\n ]\n [process where event.type == \"start\" and process.parent.name : \"winrshost.exe\" and not process.name : \"conhost.exe\"]\n", + "query": "sequence by host.id with maxspan=30s\n [network where process.pid == 4 and network.direction : (\"incoming\", \"ingress\") and\n destination.port in (5985, 5986) and network.protocol == \"http\" and not source.address in (\"::1\", \"127.0.0.1\")\n ]\n [process where event.type == \"start\" and process.parent.name : \"winrshost.exe\" and not process.name : \"conhost.exe\"]\n", "risk_score": 47, "rule_id": "1cd01db9-be24-4bef-8e7c-e923f0ff78ab", "severity": "medium", @@ -44,5 +44,5 @@ } ], "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_incoming_wmi.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_incoming_wmi.json index 901a19d896ff3..9b13ade43812f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_incoming_wmi.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_incoming_wmi.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "WMI Incoming Lateral Movement", - "query": "sequence by host.id with maxspan = 2s\n\n /* Accepted Incoming RPC connection by Winmgmt service */\n\n [network where process.name : \"svchost.exe\" and network.direction == \"incoming\" and\n source.address != \"127.0.0.1\" and source.address != \"::1\" and \n source.port >= 49152 and destination.port >= 49152\n ]\n\n /* Excluding Common FPs Nessus and SCCM */\n\n [process where event.type in (\"start\", \"process_started\") and process.parent.name : \"WmiPrvSE.exe\" and\n not process.args : (\"C:\\\\windows\\\\temp\\\\nessus_*.txt\", \n \"C:\\\\windows\\\\TEMP\\\\nessus_*.TMP\", \n \"C:\\\\Windows\\\\CCM\\\\SystemTemp\\\\*\", \n \"C:\\\\Windows\\\\CCMCache\\\\*\", \n \"C:\\\\CCM\\\\Cache\\\\*\")\n ]\n", + "query": "sequence by host.id with maxspan = 2s\n\n /* Accepted Incoming RPC connection by Winmgmt service */\n\n [network where process.name : \"svchost.exe\" and network.direction : (\"incoming\", \"ingress\") and\n source.address != \"127.0.0.1\" and source.address != \"::1\" and \n source.port >= 49152 and destination.port >= 49152\n ]\n\n /* Excluding Common FPs Nessus and SCCM */\n\n [process where event.type in (\"start\", \"process_started\") and process.parent.name : \"WmiPrvSE.exe\" and\n not process.args : (\"C:\\\\windows\\\\temp\\\\nessus_*.txt\", \n \"C:\\\\windows\\\\TEMP\\\\nessus_*.TMP\", \n \"C:\\\\Windows\\\\CCM\\\\SystemTemp\\\\*\", \n \"C:\\\\Windows\\\\CCMCache\\\\*\", \n \"C:\\\\CCM\\\\Cache\\\\*\")\n ]\n", "risk_score": 47, "rule_id": "f3475224-b179-4f78-8877-c2bd64c26b88", "severity": "medium", @@ -50,5 +50,5 @@ } ], "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_powershell_remoting_target.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_powershell_remoting_target.json index 33b5ef7c0dacb..94708f90d20bb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_powershell_remoting_target.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_powershell_remoting_target.json @@ -15,7 +15,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Incoming Execution via PowerShell Remoting", - "query": "sequence by host.id with maxspan = 30s\n [network where network.direction == \"incoming\" and destination.port in (5985, 5986) and\n network.protocol == \"http\" and source.address != \"127.0.0.1\" and source.address != \"::1\"\n ]\n [process where event.type == \"start\" and process.parent.name : \"wsmprovhost.exe\" and not process.name : \"conhost.exe\"]\n", + "query": "sequence by host.id with maxspan = 30s\n [network where network.direction : (\"incoming\", \"ingress\") and destination.port in (5985, 5986) and\n network.protocol == \"http\" and source.address != \"127.0.0.1\" and source.address != \"::1\"\n ]\n [process where event.type == \"start\" and process.parent.name : \"wsmprovhost.exe\" and not process.name : \"conhost.exe\"]\n", "references": [ "https://docs.microsoft.com/en-us/powershell/scripting/learn/remoting/running-remote-commands?view=powershell-7.1" ], @@ -47,5 +47,5 @@ } ], "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_enabled_registry.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_enabled_registry.json index 6b2f782e488c4..584f24cfb30f3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_enabled_registry.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_enabled_registry.json @@ -35,12 +35,19 @@ { "id": "T1021", "name": "Remote Services", - "reference": "https://attack.mitre.org/techniques/T1021/" + "reference": "https://attack.mitre.org/techniques/T1021/", + "subtechnique": [ + { + "id": "T1021.001", + "name": "Remote Desktop Protocol", + "reference": "https://attack.mitre.org/techniques/T1021/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_sharprdp_target.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_sharprdp_target.json index 0318883e374d3..0e5b7e7bc9001 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_sharprdp_target.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_rdp_sharprdp_target.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Potential SharpRDP Behavior", - "query": "/* Incoming RDP followed by a new RunMRU string value set to cmd, powershell, taskmgr or tsclient, followed by process execution within 1m */\n\nsequence by host.id with maxspan=1m\n [network where event.type == \"start\" and process.name : \"svchost.exe\" and destination.port == 3389 and \n network.direction == \"incoming\" and network.transport == \"tcp\" and\n source.address != \"127.0.0.1\" and source.address != \"::1\"\n ]\n\n [registry where process.name : \"explorer.exe\" and \n registry.path : (\"HKEY_USERS\\\\*\\\\Software\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\RunMRU\\\\*\") and\n registry.data.strings : (\"cmd.exe*\", \"powershell.exe*\", \"taskmgr*\", \"\\\\\\\\tsclient\\\\*.exe\\\\*\")\n ]\n \n [process where event.type in (\"start\", \"process_started\") and\n (process.parent.name : (\"cmd.exe\", \"powershell.exe\", \"taskmgr.exe\") or process.args : (\"\\\\\\\\tsclient\\\\*.exe\")) and \n not process.name : \"conhost.exe\"\n ]\n", + "query": "/* Incoming RDP followed by a new RunMRU string value set to cmd, powershell, taskmgr or tsclient, followed by process execution within 1m */\n\nsequence by host.id with maxspan=1m\n [network where event.type == \"start\" and process.name : \"svchost.exe\" and destination.port == 3389 and \n network.direction : (\"incoming\", \"ingress\") and network.transport == \"tcp\" and\n source.address != \"127.0.0.1\" and source.address != \"::1\"\n ]\n\n [registry where process.name : \"explorer.exe\" and \n registry.path : (\"HKEY_USERS\\\\*\\\\Software\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\RunMRU\\\\*\") and\n registry.data.strings : (\"cmd.exe*\", \"powershell.exe*\", \"taskmgr*\", \"\\\\\\\\tsclient\\\\*.exe\\\\*\")\n ]\n \n [process where event.type in (\"start\", \"process_started\") and\n (process.parent.name : (\"cmd.exe\", \"powershell.exe\", \"taskmgr.exe\") or process.args : (\"\\\\\\\\tsclient\\\\*.exe\")) and \n not process.name : \"conhost.exe\"\n ]\n", "references": [ "https://posts.specterops.io/revisiting-remote-desktop-lateral-movement-8fb905cb46c3", "https://github.com/sbousseaden/EVTX-ATTACK-SAMPLES/blob/master/Lateral%20Movement/LM_sysmon_3_12_13_1_SharpRDP.evtx" @@ -39,11 +39,18 @@ { "id": "T1021", "name": "Remote Services", - "reference": "https://attack.mitre.org/techniques/T1021/" + "reference": "https://attack.mitre.org/techniques/T1021/", + "subtechnique": [ + { + "id": "T1021.001", + "name": "Remote Desktop Protocol", + "reference": "https://attack.mitre.org/techniques/T1021/001/" + } + ] } ] } ], "type": "eql", - "version": 2 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_services.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_services.json index 88f5e0e63a052..5220506d37f58 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_services.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_services.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Remotely Started Services via RPC", - "query": "sequence with maxspan=1s\n [network where process.name : \"services.exe\" and\n network.direction == \"incoming\" and network.transport == \"tcp\" and \n source.port >= 49152 and destination.port >= 49152 and source.address not in (\"127.0.0.1\", \"::1\")\n ] by host.id, process.entity_id\n\n [process where event.type in (\"start\", \"process_started\") and process.parent.name : \"services.exe\" and \n not (process.name : \"svchost.exe\" and process.args : \"tiledatamodelsvc\") and \n not (process.name : \"msiexec.exe\" and process.args : \"/V\")\n \n /* uncomment if psexec is noisy in your environment */\n /* and not process.name : \"PSEXESVC.exe\" */\n ] by host.id, process.parent.entity_id\n", + "query": "sequence with maxspan=1s\n [network where process.name : \"services.exe\" and\n network.direction : (\"incoming\", \"ingress\") and network.transport == \"tcp\" and \n source.port >= 49152 and destination.port >= 49152 and source.address not in (\"127.0.0.1\", \"::1\")\n ] by host.id, process.entity_id\n\n [process where event.type in (\"start\", \"process_started\") and process.parent.name : \"services.exe\" and \n not (process.name : \"svchost.exe\" and process.args : \"tiledatamodelsvc\") and \n not (process.name : \"msiexec.exe\" and process.args : \"/V\")\n \n /* uncomment if psexec is noisy in your environment */\n /* and not process.name : \"PSEXESVC.exe\" */\n ] by host.id, process.parent.entity_id\n", "risk_score": 47, "rule_id": "aa9a274d-6b53-424d-ac5e-cb8ca4251650", "severity": "medium", @@ -41,5 +41,5 @@ } ], "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_scheduled_task_target.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_scheduled_task_target.json index b66b5a94fe27f..b60717e61765a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_scheduled_task_target.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_scheduled_task_target.json @@ -12,8 +12,8 @@ "language": "eql", "license": "Elastic License v2", "name": "Remote Scheduled Task Creation", - "note": "## Triage and analysis\n\nDecode the base64 encoded tasks actions registry value to investigate the task configured action.", - "query": "/* Task Scheduler service incoming connection followed by TaskCache registry modification */\n\nsequence by host.id, process.entity_id with maxspan = 1m\n [network where process.name : \"svchost.exe\" and\n network.direction == \"incoming\" and source.port >= 49152 and destination.port >= 49152 and\n source.address != \"127.0.0.1\" and source.address != \"::1\"\n ]\n [registry where registry.path : \"HKLM\\\\SOFTWARE\\\\Microsoft\\\\Windows NT\\\\CurrentVersion\\\\Schedule\\\\TaskCache\\\\Tasks\\\\*\\\\Actions\"]\n", + "note": "## Triage and analysis\n\n### Investigating Creation of Remote Scheduled Tasks\n\n[Scheduled tasks](https://docs.microsoft.com/en-us/windows/win32/taskschd/about-the-task-scheduler) are a great mechanism used for persistence and executing programs. These features can\nbe used remotely for a variety of legitimate reasons, but at the same time used by malware and adversaries.\nWhen investigating scheduled tasks that have been set-up remotely, one of the first methods should be determining the\noriginal intent behind the configuration and verify if the activity is tied to benign behavior such as software installations or any kind\nof network administrator work. One objective for these alerts is to understand the configured action within the scheduled\ntask, this is captured within the registry event data for this rule and can be base64 decoded to view the value.\n\n#### Possible investigation steps:\n- Review the base64 encoded tasks actions registry value to investigate the task configured action.\n- Determine if task is related to legitimate or benign behavior based on the corresponding process or program tied to the\nscheduled task.\n- Further examination should include both the source and target machines where host-based artifacts and network logs\nshould be reviewed further around the time window of the creation of the scheduled task.\n\n### False Positive Analysis\n- There is a high possibility of benign activity tied to the creation of remote scheduled tasks as it is a general feature\nwithin Windows and used for legitimate purposes for a wide range of activity. Any kind of context should be found to\nfurther understand the source of the activity and determine the intent based on the scheduled task contents.\n\n### Related Rules\n- Service Command Lateral Movement\n- Remotely Started Services via RPC\n\n### Response and Remediation\n- This behavior represents post-exploitation actions such as persistence or lateral movement, immediate response should\nbe taken to review and investigate the activity and potentially isolate involved machines to prevent further post-compromise\nbehavior.\n- Remove scheduled task and any other related artifacts to the activity.\n- Review privileged account management and user account management settings such as implementing GPO policies to further\nrestrict activity or configure settings that only allow Administrators to create remote scheduled tasks.\n", + "query": "/* Task Scheduler service incoming connection followed by TaskCache registry modification */\n\nsequence by host.id, process.entity_id with maxspan = 1m\n [network where process.name : \"svchost.exe\" and\n network.direction : (\"incoming\", \"ingress\") and source.port >= 49152 and destination.port >= 49152 and\n source.address != \"127.0.0.1\" and source.address != \"::1\"\n ]\n [registry where registry.path : \"HKLM\\\\SOFTWARE\\\\Microsoft\\\\Windows NT\\\\CurrentVersion\\\\Schedule\\\\TaskCache\\\\Tasks\\\\*\\\\Actions\"]\n", "risk_score": 47, "rule_id": "954ee7c8-5437-49ae-b2d6-2960883898e9", "severity": "medium", @@ -51,11 +51,18 @@ { "id": "T1053", "name": "Scheduled Task/Job", - "reference": "https://attack.mitre.org/techniques/T1053/" + "reference": "https://attack.mitre.org/techniques/T1053/", + "subtechnique": [ + { + "id": "T1053.005", + "name": "Scheduled Task", + "reference": "https://attack.mitre.org/techniques/T1053/005/" + } + ] } ] } ], "type": "eql", - "version": 3 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_auth_rare_user_logon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_auth_rare_user_logon.json index 2f0a60b3efba9..d5d055bfa1658 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_auth_rare_user_logon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_auth_rare_user_logon.json @@ -3,7 +3,7 @@ "author": [ "Elastic" ], - "description": "A machine learning job found an unusual user name in the authentication logs. An unusual user name is one way of detecting credentialed access by means of a new or dormant user account. A user account that is normally inactive (because the user has left the organization) that becomes active may be due to credentialed access using a compromised account password. Threat actors will sometimes also create new users as a means of persisting in a compromised web application.", + "description": "A machine learning job found an unusual user name in the authentication logs. An unusual user name is one way of detecting credentialed access by means of a new or dormant user account. An inactive user account (because the user has left the organization) that becomes active may be due to credentialed access using a compromised account password. Threat actors will sometimes also create new users as a means of persisting in a compromised web application.", "false_positives": [ "User accounts that are rarely active, such as a site reliability engineer (SRE) or developer logging into a production server for troubleshooting, may trigger this alert. Under some conditions, a newly created user account may briefly trigger this alert while the model is learning." ], @@ -25,5 +25,5 @@ "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_auth_spike_in_logon_events_from_a_source_ip.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_auth_spike_in_logon_events_from_a_source_ip.json index 8e007c96c37fb..ee9acc43ac8d7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_auth_spike_in_logon_events_from_a_source_ip.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_auth_spike_in_logon_events_from_a_source_ip.json @@ -3,7 +3,7 @@ "author": [ "Elastic" ], - "description": "A machine learning job found an unusually large spike in successful authentication events events from a particular source IP address. This can be due to password spraying, user enumeration or brute force activity.", + "description": "A machine learning job found an unusually large spike in successful authentication events from a particular source IP address. This can be due to password spraying, user enumeration or brute force activity.", "false_positives": [ "Build servers and CI systems can sometimes trigger this alert. Security test cycles that include brute force or password spraying activities may trigger this alert." ], @@ -25,5 +25,5 @@ "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json index e9ebbf2470b53..1b64f1d85301a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json @@ -12,7 +12,7 @@ "license": "Elastic License v2", "machine_learning_job_id": "high_distinct_count_error_message", "name": "Spike in AWS Error Messages", - "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\n### Investigating Spikes in CloudTrail Errors\nDetection alerts from this rule indicate a large spike in the number of CloudTrail log messages that contain a particular error message. The error message in question was associated with the response to an AWS API command or method call. Here are some possible avenues of investigation:\n- Examine the history of the error. Has it manifested before? If the error, which is visible in the `aws.cloudtrail.error_message` field, only manifested recently, it might be related to recent changes in an automation module or script.\n- Examine the request parameters. These may provide indications as to the nature of the task being performed when the error occurred. Is the error related to unsuccessful attempts to enumerate or access objects, data, or secrets? If so, this can sometimes be a byproduct of discovery, privilege escalation or lateral movement attempts.\n- Consider the user as identified by the user.name field. Is this activity part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key id in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?", + "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\n### Investigating Spikes in CloudTrail Errors\n\nCloudTrail logging provides visibility on actions taken within an AWS environment. By monitoring these events and understanding\nwhat is considered normal behavior within an organization, suspicious or malicious activity can be spotted when deviations\nare observed. This example rule triggers from a large spike in the number of CloudTrail log messages that contain a\nparticular error message. The error message in question was associated with the response to an AWS API command or method call,\nthis has the potential to uncover unknown threats or activity.\n\n#### Possible investigation steps:\n- Examine the history of the error. Has it manifested before? If the error, which is visible in the `aws.cloudtrail.error_message` field, only manifested recently, it might be related to recent changes in an automation module or script.\n- Examine the request parameters. These may provide indications as to the nature of the task being performed when the error occurred. Is the error related to unsuccessful attempts to enumerate or access objects, data, or secrets? If so, this can sometimes be a byproduct of discovery, privilege escalation or lateral movement attempts.\n- Consider the user as identified by the `user.name field`. Is this activity part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key ID in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance that's not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?\n\n### False Positive Analysis\n- This rule has the possibility to produce false positives based on unexpected activity occurring such as bugs or recent\nchanges to automation modules or scripting.\n- Adoption of new services or implementing new functionality to scripts may generate false positives\n\n### Related Rules\n- Unusual AWS Command for a User\n- Rare AWS Error Code\n\n### Response and Remediation\n- If activity is observed as suspicious or malicious, immediate response should be looked into rotating and deleting AWS IAM access keys\n- Validate if any unauthorized new users were created, remove these accounts and request password resets for other IAM users\n- Look into enabling multi-factor authentication for users\n- Follow security best practices [outlined](https://aws.amazon.com/premiumsupport/knowledge-center/security-best-practices/) by AWS\n", "references": [ "https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html" ], @@ -26,5 +26,5 @@ "ML" ], "type": "machine_learning", - "version": 6 + "version": 7 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json index ac7a867f5cd6e..d9e2b3e358760 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json @@ -12,7 +12,7 @@ "license": "Elastic License v2", "machine_learning_job_id": "rare_error_code", "name": "Rare AWS Error Code", - "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\nInvestigating Unusual CloudTrail Error Activity ###\nDetection alerts from this rule indicate a rare and unusual error code that was associated with the response to an AWS API command or method call. Here are some possible avenues of investigation:\n- Examine the history of the error. Has it manifested before? If the error, which is visible in the `aws.cloudtrail.error_code field`, only manifested recently, it might be related to recent changes in an automation module or script.\n- Examine the request parameters. These may provide indications as to the nature of the task being performed when the error occurred. Is the error related to unsuccessful attempts to enumerate or access objects, data, or secrets? If so, this can sometimes be a byproduct of discovery, privilege escalation, or lateral movement attempts.\n- Consider the user as identified by the `user.name` field. Is this activity part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key id in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?", + "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\nInvestigating Unusual CloudTrail Error Activity ###\nDetection alerts from this rule indicate a rare and unusual error code that was associated with the response to an AWS API command or method call. Here are some possible avenues of investigation:\n- Examine the history of the error. Has it manifested before? If the error, which is visible in the `aws.cloudtrail.error_code field`, only manifested recently, it might be related to recent changes in an automation module or script.\n- Examine the request parameters. These may provide indications as to the nature of the task being performed when the error occurred. Is the error related to unsuccessful attempts to enumerate or access objects, data, or secrets? If so, this can sometimes be a byproduct of discovery, privilege escalation, or lateral movement attempts.\n- Consider the user as identified by the `user.name` field. Is this activity part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key ID in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance that's not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?", "references": [ "https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html" ], @@ -26,5 +26,5 @@ "ML" ], "type": "machine_learning", - "version": 6 + "version": 7 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json index 2a31ce8c065d8..a3d6208eb9f05 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json @@ -12,7 +12,7 @@ "license": "Elastic License v2", "machine_learning_job_id": "rare_method_for_a_city", "name": "Unusual City For an AWS Command", - "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\n### Investigating an Unusual CloudTrail Event\nDetection alerts from this rule indicate an AWS API command or method call that is rare and unusual for the geolocation of the source IP address. Here are some possible avenues of investigation:\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?\n- Consider the user as identified by the `user.name` field. Is this command part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key id in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the time of day. If the user is a human, not a program or script, did the activity take place during a normal time of day?\n- Examine the history of the command. If the command, which is visible in the `event.action field`, only manifested recently, it might be part of a new automation module or script. If it has a consistent cadence (for example, if it appears in small numbers on a weekly or monthly cadence), it might be part of a housekeeping or maintenance process.\n- Examine the request parameters. These may provide indications as to the source of the program or the nature of the tasks it is performing.", + "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\n### Investigating an Unusual CloudTrail Event\nDetection alerts from this rule indicate an AWS API command or method call that is rare and unusual for the geolocation of the source IP address. Here are some possible avenues of investigation:\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance that's not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?\n- Consider the user as identified by the `user.name` field. Is this command part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key ID in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the time of day. If the user is a human, not a program or script, did the activity take place during a normal time of day?\n- Examine the history of the command. If the command, which is visible in the `event.action field`, only manifested recently, it might be part of a new automation module or script. If it has a consistent cadence (for example, if it appears in small numbers on a weekly or monthly cadence), it might be part of a housekeeping or maintenance process.\n- Examine the request parameters. These may provide indications as to the source of the program or the nature of the tasks it is performing.", "references": [ "https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html" ], @@ -26,5 +26,5 @@ "ML" ], "type": "machine_learning", - "version": 6 + "version": 7 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json index ebe7971e94289..4576b080e1ea6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json @@ -12,7 +12,7 @@ "license": "Elastic License v2", "machine_learning_job_id": "rare_method_for_a_country", "name": "Unusual Country For an AWS Command", - "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\n### Investigating an Unusual CloudTrail Event\nDetection alerts from this rule indicate an AWS API command or method call that is rare and unusual for the geolocation of the source IP address. Here are some possible avenues of investigation:\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?\n- Consider the user as identified by the `user.name` field. Is this command part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key id in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the time of day. If the user is a human, not a program or script, did the activity take place during a normal time of day?\n- Examine the history of the command. If the command, which is visible in the `event.action field`, only manifested recently, it might be part of a new automation module or script. If it has a consistent cadence (for example, if it appears in small numbers on a weekly or monthly cadence), it might be part of a housekeeping or maintenance process.\n- Examine the request parameters. These may provide indications as to the source of the program or the nature of the tasks it is performing.", + "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\n### Investigating an Unusual Country For an AWS Command\n\nCloudTrail logging provides visibility on actions taken within an AWS environment. By monitoring these events and understanding\nwhat is considered normal behavior within an organization, suspicious or malicious activity can be spotted when deviations\nare observed. This example rule focuses on AWS command activity where the country from the source of the activity has been\nconsidered unusual based on previous history.\n\n#### Possible investigation steps:\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance that's not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?\n- Consider the user as identified by the `user.name` field. Is this command part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key ID in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the time of day. If the user is a human, not a program or script, did the activity take place during a normal time of day?\n- Examine the history of the command. If the command, which is visible in the `event.action field`, only manifested recently, it might be part of a new automation module or script. If it has a consistent cadence (for example, if it appears in small numbers on a weekly or monthly cadence), it might be part of a housekeeping or maintenance process.\n- Examine the request parameters. These may provide indications as to the source of the program or the nature of the tasks it is performing.\n\n### False Positive Analysis\n- False positives can occur if activity is coming from new employees based in a country with no previous history in AWS,\ntherefore it's important to validate the activity listed in the investigation steps above.\n\n### Related Rules\n- Unusual City For an AWS Command\n- Unusual AWS Command for a User\n- Rare AWS Error Code\n\n### Response and Remediation\n- If activity is observed as suspicious or malicious, immediate response should be looked into rotating and deleting AWS IAM access keys\n- Validate if any unauthorized new users were created, remove these accounts and request password resets for other IAM users\n- Look into enabling multi-factor authentication for users\n- Follow security best practices [outlined](https://aws.amazon.com/premiumsupport/knowledge-center/security-best-practices/) by AWS\n", "references": [ "https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html" ], @@ -26,5 +26,5 @@ "ML" ], "type": "machine_learning", - "version": 6 + "version": 7 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json index ab9364c453423..53f9fab8d1b48 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json @@ -12,7 +12,7 @@ "license": "Elastic License v2", "machine_learning_job_id": "rare_method_for_a_username", "name": "Unusual AWS Command for a User", - "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\n### Investigating an Unusual CloudTrail Event\n\nDetection alerts from this rule indicate an AWS API command or method call that is rare and unusual for the calling IAM user. Here are some possible avenues of investigation:\n- Consider the user as identified by the `user.name` field. Is this command part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key id in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?\n- Consider the time of day. If the user is a human, not a program or script, did the activity take place during a normal time of day?\n- Examine the history of the command. If the command, which is visible in the `event.action field`, only manifested recently, it might be part of a new automation module or script. If it has a consistent cadence (for example, if it appears in small numbers on a weekly or monthly cadence), it might be part of a housekeeping or maintenance process.\n- Examine the request parameters. These may provide indications as to the source of the program or the nature of the tasks it is performing.", + "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.\n\n## Triage and analysis\n\n### Investigating an Unusual CloudTrail Event\n\nDetection alerts from this rule indicate an AWS API command or method call that is rare and unusual for the calling IAM user. Here are some possible avenues of investigation:\n- Consider the user as identified by the `user.name` field. Is this command part of an expected workflow for the user context? Examine the user identity in the `aws.cloudtrail.user_identity.arn` field and the access key ID in the `aws.cloudtrail.user_identity.access_key_id` field, which can help identify the precise user context. The user agent details in the `user_agent.original` field may also indicate what kind of a client made the request.\n- Consider the source IP address and geolocation for the calling user who issued the command. Do they look normal for the calling user? If the source is an EC2 IP address, is it associated with an EC2 instance in one of your accounts, or could it be sourcing from an EC2 instance that's not under your control? If it is an authorized EC2 instance, is the activity associated with normal behavior for the instance role or roles? Are there any other alerts or signs of suspicious activity involving this instance?\n- Consider the time of day. If the user is a human, not a program or script, did the activity take place during a normal time of day?\n- Examine the history of the command. If the command, which is visible in the `event.action field`, only manifested recently, it might be part of a new automation module or script. If it has a consistent cadence (for example, if it appears in small numbers on a weekly or monthly cadence), it might be part of a housekeeping or maintenance process.\n- Examine the request parameters. These may provide indications as to the source of the program or the nature of the tasks it is performing.", "references": [ "https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html" ], @@ -26,5 +26,5 @@ "ML" ], "type": "machine_learning", - "version": 6 + "version": 7 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json index 8729de9a8689d..d8bf26884b16f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json @@ -15,7 +15,7 @@ "v2_rare_process_by_host_windows_ecs" ], "name": "Unusual Process For a Windows Host", - "note": "## Triage and analysis\n\n### Investigating an Unusual Windows Process\nDetection alerts from this rule indicate the presence of a Windows process that is rare and unusual for the host it ran on. Here are some possible avenues of investigation:\n- Consider the user as identified by the username field. Is this program part of an expected workflow for the user who ran this program on this host?\n- Examine the history of execution. If this process only manifested recently, it might be part of a new software package. If it has a consistent cadence (for example if it runs monthly or quarterly), it might be part of a monthly or quarterly business process.\n- Examine the process metadata like the values of the Company, Description and Product fields which may indicate whether the program is associated with an expected software vendor or package.\n- Examine arguments and working directory. These may provide indications as to the source of the program or the nature of the tasks it is performing.\n- Consider the same for the parent process. If the parent process is a legitimate system utility or service, this could be related to software updates or system management. If the parent process is something user-facing like an Office application, this process could be more suspicious.\n- If you have file hash values in the event data, and you suspect malware, you can optionally run a search for the file hash to see if the file is identified as malware by anti-malware tools. ", + "note": "## Triage and analysis\n\n### Investigating an Unusual Windows Process\n\nSearching for abnormal Windows processes is a good methodology to find potentially malicious activity within a network.\nBy understanding what is commonly run within an environment and developing baselines for legitimate activity can help\nuncover potential malware and suspicious behaviors.\n\n#### Possible investigation steps:\n- Consider the user as identified by the `user.name` field. Is this program part of an expected workflow for the user who ran this program on this host?\n- Examine the history of execution. If this process only manifested recently, it might be part of a new software package. If it has a consistent cadence (for example if it runs monthly or quarterly), it might be part of a monthly or quarterly business process.\n- Examine the process metadata like the values of the Company, Description and Product fields which may indicate whether the program is associated with an expected software vendor or package.\n- Examine arguments and working directory. These may provide indications as to the source of the program or the nature of the tasks it is performing.\n- Consider the same for the parent process. If the parent process is a legitimate system utility or service, this could be related to software updates or system management. If the parent process is something user-facing like an Office application, this process could be more suspicious.\n- If you have file hash values in the event data, and you suspect malware, you can optionally run a search for the file hash to see if the file is identified as malware by anti-malware tools.\n\n### False Positive Analysis\n- Validate the unusual Windows process is not related to new benign software installation activity. If related to\nlegitimate software, this can be done by leveraging the exception workflow in the Kibana Security App or Elasticsearch\nAPI to tune this rule to your environment\n- Try to understand the context of the execution by thinking about the user, machine, or business purpose. It's possible that a small number of endpoints\nsuch as servers that have very unique software that might appear to be unusual, but satisfy a specific business need.\n\n### Related Rules\n- Anomalous Windows Process Creation\n- Unusual Windows Path Activity\n- Unusual Windows Process Calling the Metadata Service\n\n### Response and Remediation\n- This rule is related to process execution events and should be immediately reviewed and investigated to determine if malicious\n- Based on validation and if malicious, the impacted machine should be isolated and analyzed to determine other post-compromise\nbehavior such as setting up persistence or performing lateral movement.\n- Look into preventive measures such as Windows Defender Application Control and AppLocker to gain better control on\nwhat is allowed to run on Windows infrastructure.\n", "references": [ "https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html" ], @@ -30,5 +30,5 @@ "ML" ], "type": "machine_learning", - "version": 7 + "version": 8 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_security_group_configuration_change_detection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_security_group_configuration_change_detection.json index a3468f4a68948..b7421934ba8e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_security_group_configuration_change_detection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_security_group_configuration_change_detection.json @@ -3,7 +3,7 @@ "Elastic", "Austin Songer" ], - "description": "Identifies a change to an AWS Security Group Configuration. A security group is like a virtul firewall and modifying configurations may allow unauthorized access. Threat actors may abuse this to establish persistence, exfiltrate data, or pivot in a AWS environment.", + "description": "Identifies a change to an AWS Security Group Configuration. A security group is like a virtual firewall, and modifying configurations may allow unauthorized access. Threat actors may abuse this to establish persistence, exfiltrate data, or pivot in an AWS environment.", "false_positives": [ "A security group may be created by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. Security group creations from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule." ], @@ -49,10 +49,23 @@ "name": "Defense Evasion", "reference": "https://attack.mitre.org/tactics/TA0005/" }, - "technique": [] + "technique": [ + { + "id": "T1562", + "name": "Impair Defenses", + "reference": "https://attack.mitre.org/techniques/T1562/", + "subtechnique": [ + { + "id": "T1562.007", + "name": "Disable or Modify Cloud Firewall", + "reference": "https://attack.mitre.org/techniques/T1562/007/" + } + ] + } + ] } ], "timestamp_override": "event.ingested", "type": "query", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_evasion_hidden_local_account_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_evasion_hidden_local_account_creation.json index 8edaef5dc72fd..24f0f3d4d95b1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_evasion_hidden_local_account_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_evasion_hidden_local_account_creation.json @@ -39,12 +39,19 @@ { "id": "T1136", "name": "Create Account", - "reference": "https://attack.mitre.org/techniques/T1136/" + "reference": "https://attack.mitre.org/techniques/T1136/", + "subtechnique": [ + { + "id": "T1136.001", + "name": "Local Account", + "reference": "https://attack.mitre.org/techniques/T1136/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_evasion_registry_startup_shell_folder_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_evasion_registry_startup_shell_folder_modified.json index 947c1c748af69..21ad9c5161541 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_evasion_registry_startup_shell_folder_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_evasion_registry_startup_shell_folder_modified.json @@ -12,7 +12,7 @@ "language": "eql", "license": "Elastic License v2", "name": "Suspicious Startup Shell Folder Modification", - "note": "## Triage and analysis\n\nVerify file creation events in the new Windows Startup folder location.", + "note": "## Triage and analysis\n\n### Investigating Suspicious Startup Shell Activity\n\nTechniques used within malware and by adversaries often leverage the Windows registry to store malicious programs for\npersistence. Startup shell folders are often targeted as they are not as prevalent as normal Startup folder paths so this\nbehavior may evade existing AV/EDR solutions. Another preference is that these programs might run with higher privileges\nwhich can be ideal for an attacker.\n\n#### Possible investigation steps:\n- Review the source process and related file tied to the Windows Registry entry\n- Validate the activity is not related to planned patches, updates, network administrator activity or legitimate software\ninstallations\n- Determine if activity is unique by validating if other machines in same organization have similar entry\n\n### False Positive Analysis\n- There is a high possibility of benign legitimate programs being added to Shell folders. This activity could be based\non new software installations, patches, or any kind of network administrator related activity. Before entering further\ninvestigation, this activity should be validated that is it not related to benign activity\n\n### Related Rules\n- Startup or Run Key Registry Modification\n- Persistent Scripts in the Startup Directory\n\n### Response and Remediation\n- Activity should first be validated as a true positive event if so then immediate response should be taken to review,\ninvestigate and potentially isolate activity to prevent further post-compromise behavior\n- The respective binary or program tied to this persistence method should be further analyzed and reviewed to understand\nit's behavior and capabilities\n- Since this activity is considered post-exploitation behavior, it's important to understand how the behavior was first\ninitialized such as through a macro-enabled document that was attached in a phishing email. By understanding the source\nof the attack, this information can then be used to search for similar indicators on other machines in the same environment.\n", "query": "registry where\n registry.path : (\n \"HKLM\\\\Software\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\User Shell Folders\\\\Common Startup\",\n \"HKLM\\\\Software\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\Shell Folders\\\\Common Startup\",\n \"HKEY_USERS\\\\*\\\\Software\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\User Shell Folders\\\\Startup\",\n \"HKEY_USERS\\\\*\\\\Software\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\Shell Folders\\\\Startup\"\n ) and\n registry.data.strings != null and\n /* Normal Startup Folder Paths */\n not registry.data.strings : (\n \"C:\\\\ProgramData\\\\Microsoft\\\\Windows\\\\Start Menu\\\\Programs\\\\Startup\",\n \"%ProgramData%\\\\Microsoft\\\\Windows\\\\Start Menu\\\\Programs\\\\Startup\",\n \"%USERPROFILE%\\\\AppData\\\\Roaming\\\\Microsoft\\\\Windows\\\\Start Menu\\\\Programs\\\\Startup\",\n \"C:\\\\Users\\\\*\\\\AppData\\\\Roaming\\\\Microsoft\\\\Windows\\\\Start Menu\\\\Programs\\\\Startup\"\n )\n", "risk_score": 73, "rule_id": "c8b150f0-0164-475b-a75e-74b47800a9ff", @@ -50,5 +50,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_exchange_suspicious_mailbox_right_delegation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_exchange_suspicious_mailbox_right_delegation.json new file mode 100644 index 0000000000000..e950569f19878 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_exchange_suspicious_mailbox_right_delegation.json @@ -0,0 +1,57 @@ +{ + "author": [ + "Elastic", + "Austin Songer" + ], + "description": "Identifies the assignment of rights to accesss content from another mailbox. An adversary may use the compromised account to send messages to other accounts in the network of the target business while creating inbox rules, so messages can evade spam/phishing detection mechanisms.", + "false_positives": [ + "Assignment of rights to a service account." + ], + "index": [ + "filebeat-*", + "logs-o365*" + ], + "language": "kuery", + "license": "Elastic License v2", + "name": "O365 Exchange Suspicious Mailbox Right Delegation", + "note": "## Config\n\nThe Microsoft 365 Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.", + "query": "event.dataset:o365.audit and event.provider:Exchange and event.action:Add-MailboxPermission and \no365.audit.Parameters.AccessRights:(FullAccess or SendAs or SendOnBehalf) and event.outcome:success\n", + "risk_score": 21, + "rule_id": "0ce6487d-8069-4888-9ddd-61b52490cebc", + "severity": "low", + "tags": [ + "Elastic", + "Cloud", + "Microsoft 365", + "Continuous Monitoring", + "SecOps", + "Configuration Audit" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0003", + "name": "Persistence", + "reference": "https://attack.mitre.org/tactics/TA0003/" + }, + "technique": [ + { + "id": "T1098", + "name": "Account Manipulation", + "reference": "https://attack.mitre.org/techniques/T1098/", + "subtechnique": [ + { + "id": "T1098.002", + "name": "Exchange Email Delegate Permissions", + "reference": "https://attack.mitre.org/techniques/T1098/002/" + } + ] + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json index 86b1cd3e71eaf..ebbe2448c75df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json @@ -35,12 +35,19 @@ { "id": "T1053", "name": "Scheduled Task/Job", - "reference": "https://attack.mitre.org/techniques/T1053/" + "reference": "https://attack.mitre.org/techniques/T1053/", + "subtechnique": [ + { + "id": "T1053.005", + "name": "Scheduled Task", + "reference": "https://attack.mitre.org/techniques/T1053/005/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 5 + "version": 6 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_job_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_job_creation.json index 6e656209fd055..60afcad90333c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_job_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_job_creation.json @@ -38,12 +38,19 @@ { "id": "T1053", "name": "Scheduled Task/Job", - "reference": "https://attack.mitre.org/techniques/T1053/" + "reference": "https://attack.mitre.org/techniques/T1053/", + "subtechnique": [ + { + "id": "T1053.005", + "name": "Scheduled Task", + "reference": "https://attack.mitre.org/techniques/T1053/005/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_scripting.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_scripting.json index 712e98d4ac941..128fdd9de5575 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_scripting.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_scripting.json @@ -39,11 +39,18 @@ { "id": "T1053", "name": "Scheduled Task/Job", - "reference": "https://attack.mitre.org/techniques/T1053/" + "reference": "https://attack.mitre.org/techniques/T1053/", + "subtechnique": [ + { + "id": "T1053.005", + "name": "Scheduled Task", + "reference": "https://attack.mitre.org/techniques/T1053/005/" + } + ] } ] } ], "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_persistence_powershell_exch_mailbox_activesync_add_device.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_powershell_exch_mailbox_activesync_add_device.json similarity index 72% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_persistence_powershell_exch_mailbox_activesync_add_device.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_powershell_exch_mailbox_activesync_add_device.json index 9a494a13fa297..75044e20ca5fd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_persistence_powershell_exch_mailbox_activesync_add_device.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_powershell_exch_mailbox_activesync_add_device.json @@ -28,31 +28,33 @@ "Host", "Windows", "Threat Detection", - "Collection" + "Persistence" ], "threat": [ { "framework": "MITRE ATT&CK", "tactic": { - "id": "TA0009", - "name": "Collection", - "reference": "https://attack.mitre.org/tactics/TA0009/" + "id": "TA0003", + "name": "Persistence", + "reference": "https://attack.mitre.org/tactics/TA0003/" }, "technique": [ { - "id": "T1114", - "name": "Email Collection", - "reference": "https://attack.mitre.org/techniques/T1114/" - }, - { - "id": "T1005", - "name": "Data from Local System", - "reference": "https://attack.mitre.org/techniques/T1005/" + "id": "T1098", + "name": "Account Manipulation", + "reference": "https://attack.mitre.org/techniques/T1098/", + "subtechnique": [ + { + "id": "T1098.002", + "name": "Exchange Email Delegate Permissions", + "reference": "https://attack.mitre.org/techniques/T1098/002/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_instance_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_instance_creation.json index aa2c946d3a001..4ea6631025c11 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_instance_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_instance_creation.json @@ -1,6 +1,7 @@ { "author": [ - "Elastic" + "Elastic", + "Austin Songer" ], "description": "Identifies the creation of an Amazon Relational Database Service (RDS) Aurora database instance.", "false_positives": [ @@ -44,5 +45,5 @@ ], "timestamp_override": "event.ingested", "type": "query", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_registry_uncommon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_registry_uncommon.json index 7629ee4b821da..2b94ded55e7d4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_registry_uncommon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_registry_uncommon.json @@ -34,7 +34,20 @@ "name": "Persistence", "reference": "https://attack.mitre.org/tactics/TA0003/" }, - "technique": [] + "technique": [ + { + "id": "T1547", + "name": "Boot or Logon Autostart Execution", + "reference": "https://attack.mitre.org/techniques/T1547/", + "subtechnique": [ + { + "id": "T1547.001", + "name": "Registry Run Keys / Startup Folder", + "reference": "https://attack.mitre.org/techniques/T1547/001/" + } + ] + } + ] }, { "framework": "MITRE ATT&CK", @@ -54,5 +67,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_route_table_modified_or_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_route_table_modified_or_deleted.json new file mode 100644 index 0000000000000..54180a3a59a54 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_route_table_modified_or_deleted.json @@ -0,0 +1,55 @@ +{ + "author": [ + "Elastic", + "Austin Songer" + ], + "description": "Identifies when an AWS Route Table has been modified or deleted.", + "false_positives": [ + "Route Table could be modified or deleted by a system administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. Route Table being modified from unfamiliar users should be investigated. If known behavior is causing false positives, it can be exempted from the rule. Also automated processes that uses Terraform may lead to false positives." + ], + "from": "now-60m", + "index": [ + "filebeat-*", + "logs-aws*" + ], + "interval": "10m", + "language": "kuery", + "license": "Elastic License v2", + "name": "AWS Route Table Modified or Deleted", + "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.", + "query": "event.dataset:aws.cloudtrail and event.provider:cloudtrail.amazonaws.com and event.action:(ReplaceRoute or ReplaceRouteTableAssociation or\nDeleteRouteTable or DeleteRoute or DisassociateRouteTable) and event.outcome:success\n", + "references": [ + "https://github.com/easttimor/aws-incident-response#network-routing", + "https://docs.datadoghq.com/security_platform/default_rules/cloudtrail-aws-route-table-modified", + "https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_ReplaceRoute.html", + "https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_ReplaceRouteTableAssociation", + "https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteRouteTable.html", + "https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteRoute.html", + "https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DisassociateRouteTable.html" + ], + "risk_score": 21, + "rule_id": "e7cd5982-17c8-4959-874c-633acde7d426", + "severity": "low", + "tags": [ + "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", + "SecOps", + "Network Security" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0003", + "name": "Persistence", + "reference": "https://attack.mitre.org/tactics/TA0003/" + }, + "technique": [] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_screensaver_engine_unexpected_child_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_screensaver_engine_unexpected_child_process.json new file mode 100644 index 0000000000000..544049d2c2df1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_screensaver_engine_unexpected_child_process.json @@ -0,0 +1,50 @@ +{ + "author": [ + "Elastic" + ], + "description": "Identifies when a child process is spawned by the screensaver engine process, which is consistent with an attacker's malicious payload being executed after the screensaver activated on the endpoint. An adversary can maintain persistence on a macOS endpoint by creating a malicious screensaver (.saver) file and configuring the screensaver plist file to execute code each time the screensaver is activated.", + "from": "now-9m", + "index": [ + "auditbeat-*", + "logs-endpoint.events.*" + ], + "language": "eql", + "license": "Elastic License v2", + "name": "Unexpected Child Process of macOS Screensaver Engine", + "note": "## Triage and analysis\n\n- Analyze the descendant processes of the ScreenSaverEngine process for malicious code and suspicious behavior such\nas downloading a payload from a server\n- Review the installed and activated screensaver on the host. Triage the screensaver (.saver) file that was triggered to\nidentify whether the file is malicious or not.\n", + "query": "process where event.type == \"start\" and process.parent.name == \"ScreenSaverEngine\"\n", + "references": [ + "https://posts.specterops.io/saving-your-access-d562bf5bf90b", + "https://github.com/D00MFist/PersistentJXA" + ], + "risk_score": 47, + "rule_id": "48d7f54d-c29e-4430-93a9-9db6b5892270", + "severity": "medium", + "tags": [ + "Elastic", + "Host", + "macOS", + "Threat Detection", + "Persistence" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0003", + "name": "Persistence", + "reference": "https://attack.mitre.org/tactics/TA0003/" + }, + "technique": [ + { + "id": "T1546", + "name": "Event Triggered Execution", + "reference": "https://attack.mitre.org/techniques/T1546/" + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_screensaver_plist_file_modification.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_screensaver_plist_file_modification.json new file mode 100644 index 0000000000000..dcd7427d7cbcd --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_screensaver_plist_file_modification.json @@ -0,0 +1,50 @@ +{ + "author": [ + "Elastic" + ], + "description": "Identifies when a screensaver plist file is modified by an unexpected process. An adversary can maintain persistence on a macOS endpoint by creating a malicious screensaver (.saver) file and configuring the screensaver plist file to execute code each time the screensaver is activated.", + "from": "now-9m", + "index": [ + "auditbeat-*", + "logs-endpoint.events.*" + ], + "language": "eql", + "license": "Elastic License v2", + "name": "Screensaver Plist File Modified by Unexpected Process", + "note": "## Triage and analysis\n\n- Analyze the plist file modification event to identify whether the change was expected or not\n- Investigate the process that modified the plist file for malicious code or other suspicious behavior\n- Identify if any suspicious or known malicious screensaver (.saver) files were recently written to or modified on the host", + "query": "file where event.type != \"deletion\" and\n file.name: \"com.apple.screensaver.*.plist\" and\n file.path : (\n \"/Users/*/Library/Preferences/ByHost/*\",\n \"/Library/Managed Preferences/*\",\n \"/System/Library/Preferences/*\"\n ) and\n /* Filter OS processes modifying screensaver plist files */\n not process.executable : (\n \"/usr/sbin/cfprefsd\",\n \"/usr/libexec/xpcproxy\",\n \"/System/Library/CoreServices/ManagedClient.app/Contents/Resources/MCXCompositor\",\n \"/System/Library/CoreServices/ManagedClient.app/Contents/MacOS/ManagedClient\"\n )\n", + "references": [ + "https://posts.specterops.io/saving-your-access-d562bf5bf90b", + "https://github.com/D00MFist/PersistentJXA" + ], + "risk_score": 47, + "rule_id": "e6e8912f-283f-4d0d-8442-e0dcaf49944b", + "severity": "medium", + "tags": [ + "Elastic", + "Host", + "macOS", + "Threat Detection", + "Persistence" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0003", + "name": "Persistence", + "reference": "https://attack.mitre.org/tactics/TA0003/" + }, + "technique": [ + { + "id": "T1546", + "name": "Event Triggered Execution", + "reference": "https://attack.mitre.org/techniques/T1546/" + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_suspicious_scheduled_task_runtime.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_suspicious_scheduled_task_runtime.json index ea5917a246afe..812c35350677f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_suspicious_scheduled_task_runtime.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_suspicious_scheduled_task_runtime.json @@ -38,12 +38,19 @@ { "id": "T1053", "name": "Scheduled Task/Job", - "reference": "https://attack.mitre.org/techniques/T1053/" + "reference": "https://attack.mitre.org/techniques/T1053/", + "subtechnique": [ + { + "id": "T1053.005", + "name": "Scheduled Task", + "reference": "https://attack.mitre.org/techniques/T1053/005/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_added_to_privileged_group_ad.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_added_to_privileged_group_ad.json index c63d96b106a01..1e55f014806f3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_added_to_privileged_group_ad.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_added_to_privileged_group_ad.json @@ -36,21 +36,14 @@ }, "technique": [ { - "id": "T1136", - "name": "Create Account", - "reference": "https://attack.mitre.org/techniques/T1136/", - "subtechnique": [ - { - "id": "T1136.001", - "name": "Local Account", - "reference": "https://attack.mitre.org/techniques/T1136/001/" - } - ] + "id": "T1098", + "name": "Account Manipulation", + "reference": "https://attack.mitre.org/techniques/T1098/" } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json index 0e2b01a1967d2..0777dfccab4bf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json @@ -35,12 +35,19 @@ { "id": "T1136", "name": "Create Account", - "reference": "https://attack.mitre.org/techniques/T1136/" + "reference": "https://attack.mitre.org/techniques/T1136/", + "subtechnique": [ + { + "id": "T1136.001", + "name": "Local Account", + "reference": "https://attack.mitre.org/techniques/T1136/001/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 8 + "version": 9 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json index dca20728b40fa..0d9cd0cb4020a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json @@ -38,12 +38,19 @@ { "id": "T1053", "name": "Scheduled Task/Job", - "reference": "https://attack.mitre.org/techniques/T1053/" + "reference": "https://attack.mitre.org/techniques/T1053/", + "subtechnique": [ + { + "id": "T1053.005", + "name": "Scheduled Task", + "reference": "https://attack.mitre.org/techniques/T1053/005/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 5 + "version": 6 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_windows_management_instrumentation_event_subscription.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_windows_management_instrumentation_event_subscription.json index fc3d94498d0cb..79e887a548bcb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_windows_management_instrumentation_event_subscription.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_windows_management_instrumentation_event_subscription.json @@ -35,12 +35,19 @@ { "id": "T1546", "name": "Event Triggered Execution", - "reference": "https://attack.mitre.org/techniques/T1546/" + "reference": "https://attack.mitre.org/techniques/T1546/", + "subtechnique": [ + { + "id": "T1546.003", + "name": "Windows Management Instrumentation Event Subscription", + "reference": "https://attack.mitre.org/techniques/T1546/003/" + } + ] } ] } ], "timestamp_override": "event.ingested", "type": "eql", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_webshell_detection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_webshell_detection.json index 26248009f5a49..8da3be0b69d91 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_webshell_detection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_webshell_detection.json @@ -4,7 +4,7 @@ ], "description": "Identifies suspicious commands executed via a web server, which may suggest a vulnerability and remote shell access.", "false_positives": [ - "Security audits, maintenance and network administrative scripts may trigger this alert when run under web processes." + "Security audits, maintenance, and network administrative scripts may trigger this alert when run under web processes." ], "from": "now-9m", "index": [ @@ -71,5 +71,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_new_or_modified_federation_domain.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_new_or_modified_federation_domain.json new file mode 100644 index 0000000000000..2a1231e96d8a5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_new_or_modified_federation_domain.json @@ -0,0 +1,61 @@ +{ + "author": [ + "Austin Songer" + ], + "description": "Identifies a new or modified federation domain, which can be used to create a trust between O365 and an external identity provider.", + "index": [ + "filebeat-*", + "logs-o365*" + ], + "language": "kuery", + "license": "Elastic License v2", + "name": "New or Modified Federation Domain", + "note": "## Config\n\nThe Microsoft 365 Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.", + "query": "event.dataset:o365.audit and event.provider:Exchange and event.category:web and event.action:(\"Set-AcceptedDomain\" or \n\"Set-MsolDomainFederationSettings\" or \"Add-FederatedDomain\" or \"New-AcceptedDomain\" or \"Remove-AcceptedDomain\" or \"Remove-FederatedDomain\") and \nevent.outcome:success\n", + "references": [ + "https://docs.microsoft.com/en-us/powershell/module/exchange/remove-accepteddomain?view=exchange-ps", + "https://docs.microsoft.com/en-us/powershell/module/exchange/remove-federateddomain?view=exchange-ps", + "https://docs.microsoft.com/en-us/powershell/module/exchange/new-accepteddomain?view=exchange-ps", + "https://docs.microsoft.com/en-us/powershell/module/exchange/add-federateddomain?view=exchange-ps", + "https://docs.microsoft.com/en-us/powershell/module/exchange/set-accepteddomain?view=exchange-ps", + "https://docs.microsoft.com/en-us/powershell/module/msonline/set-msoldomainfederationsettings?view=azureadps-1.0" + ], + "risk_score": 21, + "rule_id": "684554fc-0777-47ce-8c9b-3d01f198d7f8", + "severity": "low", + "tags": [ + "Elastic", + "Cloud", + "Microsoft 365", + "Continuous Monitoring", + "SecOps", + "Identity and Access" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0004", + "name": "Privilege Escalation", + "reference": "https://attack.mitre.org/tactics/TA0004/" + }, + "technique": [ + { + "id": "T1484", + "name": "Domain Policy Modification", + "reference": "https://attack.mitre.org/techniques/T1484/", + "subtechnique": [ + { + "id": "T1484.002", + "name": "Domain Trust Modification", + "reference": "https://attack.mitre.org/techniques/T1484/002/" + } + ] + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sts_getsessiontoken_abuse.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sts_getsessiontoken_abuse.json new file mode 100644 index 0000000000000..c5e2669c1fade --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sts_getsessiontoken_abuse.json @@ -0,0 +1,74 @@ +{ + "author": [ + "Austin Songer" + ], + "description": "Identifies the suspicious use of GetSessionToken. Tokens could be created and used by attackers to move laterally and escalate privileges.", + "false_positives": [ + "GetSessionToken may be done by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. GetSessionToken from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule." + ], + "index": [ + "filebeat-*", + "logs-aws*" + ], + "language": "kuery", + "license": "Elastic License v2", + "name": "AWS STS GetSessionToken Abuse", + "note": "## Config\n\nThe AWS Fleet integration, Filebeat module, or similarly structured data is required to be compatible with this rule.", + "query": "event.dataset:aws.cloudtrail and event.provider:sts.amazonaws.com and event.action:GetSessionToken and \naws.cloudtrail.user_identity.type:IAMUser and event.outcome:success\n", + "references": [ + "https://docs.aws.amazon.com/STS/latest/APIReference/API_GetSessionToken.html" + ], + "risk_score": 21, + "rule_id": "b45ab1d2-712f-4f01-a751-df3826969807", + "severity": "low", + "tags": [ + "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", + "SecOps", + "Identity and Access" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0004", + "name": "Privilege Escalation", + "reference": "https://attack.mitre.org/tactics/TA0004/" + }, + "technique": [ + { + "id": "T1548", + "name": "Abuse Elevation Control Mechanism", + "reference": "https://attack.mitre.org/techniques/T1548/" + } + ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0008", + "name": "Lateral Movement", + "reference": "https://attack.mitre.org/tactics/TA0008/" + }, + "technique": [ + { + "id": "T1550", + "name": "Use Alternate Authentication Material", + "reference": "https://attack.mitre.org/techniques/T1550/", + "subtechnique": [ + { + "id": "T1550.001", + "name": "Application Access Token", + "reference": "https://attack.mitre.org/techniques/T1550/001/" + } + ] + } + ] + } + ], + "timestamp_override": "event.ingested", + "type": "query", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json index 9cdf474efb450..d9b7280392f38 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json @@ -14,7 +14,7 @@ "name": "Unusual Parent-Child Relationship", "query": "process where event.type in (\"start\", \"process_started\") and\nprocess.parent.name != null and\n (\n /* suspicious parent processes */\n (process.name:\"autochk.exe\" and not process.parent.name:\"smss.exe\") or\n (process.name:(\"fontdrvhost.exe\", \"dwm.exe\") and not process.parent.name:(\"wininit.exe\", \"winlogon.exe\")) or\n (process.name:(\"consent.exe\", \"RuntimeBroker.exe\", \"TiWorker.exe\") and not process.parent.name:\"svchost.exe\") or\n (process.name:\"SearchIndexer.exe\" and not process.parent.name:\"services.exe\") or\n (process.name:\"SearchProtocolHost.exe\" and not process.parent.name:(\"SearchIndexer.exe\", \"dllhost.exe\")) or\n (process.name:\"dllhost.exe\" and not process.parent.name:(\"services.exe\", \"svchost.exe\")) or\n (process.name:\"smss.exe\" and not process.parent.name:(\"System\", \"smss.exe\")) or\n (process.name:\"csrss.exe\" and not process.parent.name:(\"smss.exe\", \"svchost.exe\")) or\n (process.name:\"wininit.exe\" and not process.parent.name:\"smss.exe\") or\n (process.name:\"winlogon.exe\" and not process.parent.name:\"smss.exe\") or\n (process.name:(\"lsass.exe\", \"LsaIso.exe\") and not process.parent.name:\"wininit.exe\") or\n (process.name:\"LogonUI.exe\" and not process.parent.name:(\"wininit.exe\", \"winlogon.exe\")) or\n (process.name:\"services.exe\" and not process.parent.name:\"wininit.exe\") or\n (process.name:\"svchost.exe\" and not process.parent.name:(\"MsMpEng.exe\", \"services.exe\")) or\n (process.name:\"spoolsv.exe\" and not process.parent.name:\"services.exe\") or\n (process.name:\"taskhost.exe\" and not process.parent.name:(\"services.exe\", \"svchost.exe\")) or\n (process.name:\"taskhostw.exe\" and not process.parent.name:(\"services.exe\", \"svchost.exe\")) or\n (process.name:\"userinit.exe\" and not process.parent.name:(\"dwm.exe\", \"winlogon.exe\")) or\n (process.name:(\"wmiprvse.exe\", \"wsmprovhost.exe\", \"winrshost.exe\") and not process.parent.name:\"svchost.exe\") or\n /* suspicious child processes */\n (process.parent.name:(\"SearchProtocolHost.exe\", \"taskhost.exe\", \"csrss.exe\") and not process.name:(\"werfault.exe\", \"wermgr.exe\", \"WerFaultSecure.exe\")) or\n (process.parent.name:\"autochk.exe\" and not process.name:(\"chkdsk.exe\", \"doskey.exe\", \"WerFault.exe\")) or\n (process.parent.name:\"smss.exe\" and not process.name:(\"autochk.exe\", \"smss.exe\", \"csrss.exe\", \"wininit.exe\", \"winlogon.exe\", \"setupcl.exe\", \"WerFault.exe\")) or\n (process.parent.name:\"wermgr.exe\" and not process.name:(\"WerFaultSecure.exe\", \"wermgr.exe\", \"WerFault.exe\")) or\n (process.parent.name:\"conhost.exe\" and not process.name:(\"mscorsvw.exe\", \"wermgr.exe\", \"WerFault.exe\", \"WerFaultSecure.exe\"))\n )\n", "references": [ - "https://github.com/sbousseaden/Slides/blob/master/Hunting MindMaps/PNG/Windows Processes%20TH.map.png", + "https://github.com/sbousseaden/Slides/blob/master/Hunting%20MindMaps/PNG/Windows%20Processes%20TH.map.png", "https://www.andreafortuna.org/2017/06/15/standard-windows-processes-a-brief-reference/" ], "risk_score": 47, @@ -53,5 +53,5 @@ ], "timestamp_override": "event.ingested", "type": "eql", - "version": 9 + "version": 10 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/threat_intel_module_match.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/threat_intel_module_match.json index f582eba053d64..e4b1309c42644 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/threat_intel_module_match.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/threat_intel_module_match.json @@ -16,7 +16,7 @@ "language": "kuery", "license": "Elastic License v2", "name": "Threat Intel Filebeat Module Indicator Match", - "note": "## Triage and Analysis\n\nIf an indicator matches a local observation, the following enriched fields will be generated to identify the indicator, field, and type matched.\n\n- `threatintel.indicator.matched.atomic` - this identifies the atomic indicator that matched the local observation\n- `threatintel.indicator.matched.field` - this identifies the indicator field that matched the local observation\n- `threatintel.indicator.matched.type` - this identifies the indicator type that matched the local observation\n", + "note": "## Triage and Analysis\n\n### Investigating Threat Intel Indicator Matches\n\nThreat Intel indicator match rules allow matching from a local observation such as an endpoint event that records a file\nhash with an entry of a file hash stored within the Threat Intel Filebeat module. Other examples of matches can occur on\nan IP address, registry path, URL and imphash.\n\nThe matches will be based on the incoming feed data so it's important to validate the data and review the results by\ninvestigating the associated activity to determine if it requires further investigation.\n\nIf an indicator matches a local observation, the following enriched fields will be generated to identify the indicator, field, and type matched.\n\n- `threatintel.indicator.matched.atomic` - this identifies the atomic indicator that matched the local observation\n- `threatintel.indicator.matched.field` - this identifies the indicator field that matched the local observation\n- `threatintel.indicator.matched.type` - this identifies the indicator type that matched the local observation\n\n#### Possible investigation steps:\n- Investigation should be validated and reviewed based on the data (file hash, registry path, URL, imphash) that was matched\nand viewing the source of that activity.\n- Consider the history of the indicator that was matched. Has it happened before? Is it happening on multiple machines?\nThese kinds of questions can help understand if the activity is related to legitimate behavior.\n- Consider the user and their role within the company, is this something related to their job or work function?\n\n### False Positive Analysis\n- For any matches found, it's important to consider the initial release date of that indicator. Threat intelligence can\nbe a great tool for augmenting existing security processes, while at the same time it should be understood that threat\nintelligence can represent a specific set of activity observed at a point in time. For example, an IP address\nmay have hosted malware observed in a Dridex campaign six months ago, but it's possible that IP has been remediated and\nno longer represents any threat.\n- Adversaries often use legitimate tools as network administrators such as `PsExec` or `AdFind`, these tools often find their\nway into indicator lists creating the potential for false positives.\n- It's possible after large and publicly written campaigns, curious employees might end up going directly to attacker infrastructure and generating these rules\n\n### Response and Remediation\n- If suspicious or malicious behavior is observed, immediate response should be taken to isolate activity to prevent further\npost-compromise behavior.\n- One example of a response if a machine matched a command and control IP address would be to add an entry to a network\ndevice such as a firewall or proxy appliance to prevent any outbound activity from leaving that machine.\n- Another example of a response with a malicious file hash match would involve validating if the file was properly quarantined,\nreview current running processes looking for any abnormal activity, and investigating for any other follow-up actions such as persistence or lateral movement\n", "query": "file.hash.*:* or file.pe.imphash:* or source.ip:* or destination.ip:* or url.full:* or registry.path:*\n", "references": [ "https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-threatintel.html" @@ -190,9 +190,9 @@ ] } ], - "threat_query": "event.module:threatintel and (threatintel.indicator.file.hash.*:* or threatintel.indicator.file.pe.imphash:* or threatintel.indicator.ip:* or threatintel.indicator.registry.path:* or threatintel.indicator.url.full:*)", + "threat_query": "@timestamp >= \"now-30d\" and event.module:threatintel and (threatintel.indicator.file.hash.*:* or threatintel.indicator.file.pe.imphash:* or threatintel.indicator.ip:* or threatintel.indicator.registry.path:* or threatintel.indicator.url.full:*)", "timeline_id": "495ad7a7-316e-4544-8a0f-9c098daee76e", "timeline_title": "Generic Threat Match Timeline", "type": "threat_match", - "version": 1 + "version": 2 }