From 7688fe3b58a05603919e65257dbbf6b0ba1241a2 Mon Sep 17 00:00:00 2001 From: Luke Gmys Date: Tue, 27 Sep 2022 14:58:33 +0200 Subject: [PATCH 1/4] [TIP] Loading state changes and react-query integration (#141102) --- .../public/common/mocks/story_providers.tsx | 21 +- .../public/common/mocks/test_providers.tsx | 31 ++- .../public/common/utils/barchart.test.ts | 2 +- .../public/common/utils/barchart.ts | 4 +- .../indicators_barchart.stories.tsx | 2 +- .../indicators_barchart.test.tsx | 2 +- .../indicators_barchart.tsx | 2 +- .../indicators_barchart_wrapper.stories.tsx | 2 +- .../indicators_table/cell_actions.tsx | 2 +- .../indicators_table.stories.tsx | 4 +- .../indicators_table.test.tsx | 6 +- .../indicators_table/indicators_table.tsx | 13 +- .../hooks/use_aggregated_indicators.test.tsx | 176 +++++------- .../hooks/use_aggregated_indicators.ts | 197 +++---------- .../indicators/hooks/use_indicators.test.tsx | 260 ++++-------------- .../indicators/hooks/use_indicators.ts | 186 ++++--------- .../hooks/use_indicators_total_count.tsx | 2 +- .../indicators/indicators_page.test.tsx | 3 +- .../modules/indicators/indicators_page.tsx | 70 ++--- .../fetch_aggregated_indicators.test.ts | 117 ++++++++ .../services/fetch_aggregated_indicators.ts | 125 +++++++++ .../services/fetch_indicators.test.ts | 105 +++++++ .../indicators/services/fetch_indicators.ts | 84 ++++++ .../utils/get_indicator_query_params.ts | 70 +++++ .../indicators/utils/get_indicators_query.ts | 50 ---- .../public/modules/indicators/utils/search.ts | 75 +++++ .../components/query_bar/query_bar.tsx | 13 +- .../scripts/generate_indicators.js | 4 +- 28 files changed, 894 insertions(+), 734 deletions(-) create mode 100644 x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_aggregated_indicators.test.ts create mode 100644 x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_aggregated_indicators.ts create mode 100644 x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_indicators.test.ts create mode 100644 x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_indicators.ts create mode 100644 x-pack/plugins/threat_intelligence/public/modules/indicators/utils/get_indicator_query_params.ts delete mode 100644 x-pack/plugins/threat_intelligence/public/modules/indicators/utils/get_indicators_query.ts create mode 100644 x-pack/plugins/threat_intelligence/public/modules/indicators/utils/search.ts diff --git a/x-pack/plugins/threat_intelligence/public/common/mocks/story_providers.tsx b/x-pack/plugins/threat_intelligence/public/common/mocks/story_providers.tsx index 78c064f72b44..eea2596327fb 100644 --- a/x-pack/plugins/threat_intelligence/public/common/mocks/story_providers.tsx +++ b/x-pack/plugins/threat_intelligence/public/common/mocks/story_providers.tsx @@ -11,6 +11,8 @@ import { DataPublicPluginStart } from '@kbn/data-plugin/public'; import { CoreStart, IUiSettingsClient } from '@kbn/core/public'; import { TimelinesUIStart } from '@kbn/timelines-plugin/public'; import { EuiThemeProvider } from '@kbn/kibana-react-plugin/common'; +import { RequestAdapter } from '@kbn/inspector-plugin/common'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { mockIndicatorsFiltersContext } from './mock_indicators_filters_context'; import { SecuritySolutionContext } from '../../containers/security_solution_context'; import { getSecuritySolutionContextMock } from './mock_security_context'; @@ -20,6 +22,7 @@ import { generateFieldTypeMap } from './mock_field_type_map'; import { mockUiSettingsService } from './mock_kibana_ui_settings_service'; import { mockKibanaTimelinesService } from './mock_kibana_timelines_service'; import { mockTriggersActionsUiService } from './mock_kibana_triggers_actions_ui_service'; +import { InspectorContext } from '../../containers/inspector'; export interface KibanaContextMock { /** @@ -81,13 +84,17 @@ export const StoryProvidersComponent: VFC = ({ return ( - - - - {children} - - - + + + + + + {children} + + + + + ); }; diff --git a/x-pack/plugins/threat_intelligence/public/common/mocks/test_providers.tsx b/x-pack/plugins/threat_intelligence/public/common/mocks/test_providers.tsx index 7e046e214b54..a93b6bfe3046 100644 --- a/x-pack/plugins/threat_intelligence/public/common/mocks/test_providers.tsx +++ b/x-pack/plugins/threat_intelligence/public/common/mocks/test_providers.tsx @@ -17,6 +17,7 @@ import { unifiedSearchPluginMock } from '@kbn/unified-search-plugin/public/mocks import { createTGridMocks } from '@kbn/timelines-plugin/public/mock'; import { EuiThemeProvider } from '@kbn/kibana-react-plugin/common'; import { RequestAdapter } from '@kbn/inspector-plugin/common'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { KibanaContext } from '../../hooks/use_kibana'; import { SecuritySolutionPluginContext } from '../../types'; import { getSecuritySolutionContextMock } from './mock_security_context'; @@ -128,27 +129,31 @@ export const mockedServices = { export const TestProvidersComponent: FC = ({ children }) => ( - - - - - - - {children} - - - - - - + + + + + + + + {children} + + + + + + + ); export type MockedSearch = jest.Mocked; export type MockedTimefilter = jest.Mocked; export type MockedTriggersActionsUi = jest.Mocked; +export type MockedQueryService = jest.Mocked; export const mockedSearchService = mockedServices.data.search as MockedSearch; +export const mockedQueryService = mockedServices.data.query as MockedQueryService; export const mockedTimefilterService = mockedServices.data.query.timefilter as MockedTimefilter; export const mockedTriggersActionsUiService = mockedServices.triggersActionsUi as MockedTriggersActionsUi; diff --git a/x-pack/plugins/threat_intelligence/public/common/utils/barchart.test.ts b/x-pack/plugins/threat_intelligence/public/common/utils/barchart.test.ts index 1acf8d021334..004071059a73 100644 --- a/x-pack/plugins/threat_intelligence/public/common/utils/barchart.test.ts +++ b/x-pack/plugins/threat_intelligence/public/common/utils/barchart.test.ts @@ -5,8 +5,8 @@ * 2.0. */ +import type { Aggregation } from '../../modules/indicators/services/fetch_aggregated_indicators'; import { convertAggregationToChartSeries } from './barchart'; -import { Aggregation } from '../../modules/indicators/hooks/use_aggregated_indicators'; const aggregation1: Aggregation = { events: { diff --git a/x-pack/plugins/threat_intelligence/public/common/utils/barchart.ts b/x-pack/plugins/threat_intelligence/public/common/utils/barchart.ts index 93f6b4ce6fd6..c994e7e9f3a3 100644 --- a/x-pack/plugins/threat_intelligence/public/common/utils/barchart.ts +++ b/x-pack/plugins/threat_intelligence/public/common/utils/barchart.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { +import type { Aggregation, AggregationValue, ChartSeries, -} from '../../modules/indicators/hooks/use_aggregated_indicators'; +} from '../../modules/indicators/services/fetch_aggregated_indicators'; /** * Converts data received from an Elastic search with date_histogram aggregation enabled to something usable in the "@elastic/chart" BarChart component diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.stories.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.stories.tsx index 465ef3bd1ab7..bb0a1b1205b5 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.stories.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.stories.tsx @@ -11,8 +11,8 @@ import { Story } from '@storybook/react'; import { TimeRangeBounds } from '@kbn/data-plugin/common'; import { StoryProvidersComponent } from '../../../../common/mocks/story_providers'; import { mockKibanaTimelinesService } from '../../../../common/mocks/mock_kibana_timelines_service'; -import { ChartSeries } from '../../hooks/use_aggregated_indicators'; import { IndicatorsBarChart } from './indicators_barchart'; +import { ChartSeries } from '../../services/fetch_aggregated_indicators'; const mockIndicators: ChartSeries[] = [ { diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.test.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.test.tsx index 38de7df2b3d4..19542bdf200a 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.test.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.test.tsx @@ -10,8 +10,8 @@ import React from 'react'; import { render } from '@testing-library/react'; import { TimeRangeBounds } from '@kbn/data-plugin/common'; import { TestProvidersComponent } from '../../../../common/mocks/test_providers'; -import { ChartSeries } from '../../hooks/use_aggregated_indicators'; import { IndicatorsBarChart } from './indicators_barchart'; +import { ChartSeries } from '../../services/fetch_aggregated_indicators'; moment.suppressDeprecationWarnings = true; moment.tz.setDefault('UTC'); diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.tsx index 75d731b23c3b..d5535f53a862 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart/indicators_barchart.tsx @@ -11,7 +11,7 @@ import { EuiThemeProvider } from '@elastic/eui'; import { TimeRangeBounds } from '@kbn/data-plugin/common'; import { IndicatorBarchartLegendAction } from '../indicator_barchart_legend_action/indicator_barchart_legend_action'; import { barChartTimeAxisLabelFormatter } from '../../../../common/utils/dates'; -import { ChartSeries } from '../../hooks/use_aggregated_indicators'; +import type { ChartSeries } from '../../services/fetch_aggregated_indicators'; const ID = 'tiIndicator'; const DEFAULT_CHART_HEIGHT = '200px'; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/indicators_barchart_wrapper.stories.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/indicators_barchart_wrapper.stories.tsx index a9eec2afcf19..213c750c5d1e 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/indicators_barchart_wrapper.stories.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/indicators_barchart_wrapper.stories.tsx @@ -16,9 +16,9 @@ import { DataPublicPluginStart } from '@kbn/data-plugin/public'; import { IUiSettingsClient } from '@kbn/core/public'; import { StoryProvidersComponent } from '../../../../common/mocks/story_providers'; import { mockKibanaTimelinesService } from '../../../../common/mocks/mock_kibana_timelines_service'; -import { Aggregation, AGGREGATION_NAME } from '../../hooks/use_aggregated_indicators'; import { DEFAULT_TIME_RANGE } from '../../../query_bar/hooks/use_filters/utils'; import { IndicatorsBarChartWrapper } from './indicators_barchart_wrapper'; +import { Aggregation, AGGREGATION_NAME } from '../../services/fetch_aggregated_indicators'; export default { component: IndicatorsBarChartWrapper, diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/cell_actions.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/cell_actions.tsx index 38f22fe34556..d10ba709bfa2 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/cell_actions.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/cell_actions.tsx @@ -9,11 +9,11 @@ import React, { VFC } from 'react'; import { EuiDataGridColumnCellActionProps } from '@elastic/eui/src/components/datagrid/data_grid_types'; import { ComponentType } from '../../../../../common/types/component_type'; import { Indicator } from '../../../../../common/types/indicator'; -import { Pagination } from '../../hooks/use_indicators'; import { AddToTimeline } from '../../../timeline/components/add_to_timeline'; import { fieldAndValueValid, getIndicatorFieldAndValue } from '../../utils/field_value'; import { FilterIn } from '../../../query_bar/components/filter_in'; import { FilterOut } from '../../../query_bar/components/filter_out'; +import type { Pagination } from '../../services/fetch_indicators'; export const CELL_TIMELINE_BUTTON_TEST_ID = 'tiIndicatorsTableCellTimelineButton'; export const CELL_FILTER_IN_BUTTON_TEST_ID = 'tiIndicatorsTableCellFilterInButton'; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.stories.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.stories.tsx index 1d563141952e..034fc1363043 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.stories.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.stories.tsx @@ -44,7 +44,7 @@ export function WithIndicators() { diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.test.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.test.tsx index 71786878bdae..b110c0f91c31 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.test.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.test.tsx @@ -22,7 +22,7 @@ const tableProps: IndicatorsTableProps = { indicators: [], pagination: { pageSize: 10, pageIndex: 0, pageSizeOptions: [10] }, indicatorCount: 0, - loading: false, + isLoading: false, browserFields: {}, indexPattern: { fields: [], title: '' } as SecuritySolutionDataViewBase, columnSettings: { @@ -60,7 +60,7 @@ describe('', () => { await act(async () => { render( - + ); }); @@ -74,7 +74,7 @@ describe('', () => { diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.tsx index 3e3035c8c43e..f12e080b000b 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/indicators_table.tsx @@ -24,11 +24,11 @@ import { cellRendererFactory } from './cell_renderer'; import { EmptyState } from '../../../../components/empty_state'; import { IndicatorsTableContext, IndicatorsTableContextValue } from './context'; import { IndicatorsFlyout } from '../indicators_flyout/indicators_flyout'; -import { Pagination } from '../../hooks/use_indicators'; import { useToolbarOptions } from './hooks/use_toolbar_options'; import { ColumnSettingsValue } from './hooks/use_column_settings'; import { useFieldTypes } from '../../../../hooks/use_field_types'; import { getFieldSchema } from '../../utils/get_field_schema'; +import { Pagination } from '../../services/fetch_indicators'; export interface IndicatorsTableProps { indicators: Indicator[]; @@ -36,7 +36,10 @@ export interface IndicatorsTableProps { pagination: Pagination; onChangeItemsPerPage: (value: number) => void; onChangePage: (value: number) => void; - loading: boolean; + /** + * If true, no data is available yet + */ + isLoading: boolean; indexPattern: SecuritySolutionDataViewBase; browserFields: BrowserFields; columnSettings: ColumnSettingsValue; @@ -57,7 +60,7 @@ export const IndicatorsTable: VFC = ({ onChangePage, onChangeItemsPerPage, pagination, - loading, + isLoading, browserFields, columnSettings: { columns, columnVisibility, handleResetColumns, handleToggleColumn, sorting }, }) => { @@ -137,7 +140,7 @@ export const IndicatorsTable: VFC = ({ ); const gridFragment = useMemo(() => { - if (loading) { + if (isLoading) { return ( @@ -177,7 +180,7 @@ export const IndicatorsTable: VFC = ({ mappedColumns, indicatorCount, leadingControlColumns, - loading, + isLoading, onChangeItemsPerPage, onChangePage, pagination, diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_aggregated_indicators.test.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_aggregated_indicators.test.tsx index be8ea27374a1..6b3feb740690 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_aggregated_indicators.test.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_aggregated_indicators.test.tsx @@ -5,34 +5,18 @@ * 2.0. */ -import moment from 'moment'; -import { BehaviorSubject, throwError } from 'rxjs'; -import { renderHook } from '@testing-library/react-hooks'; -import { IKibanaSearchResponse, TimeRangeBounds } from '@kbn/data-plugin/common'; -import { - AGGREGATION_NAME, - RawAggregatedIndicatorsResponse, - useAggregatedIndicators, - UseAggregatedIndicatorsParam, -} from './use_aggregated_indicators'; +import { act, renderHook } from '@testing-library/react-hooks'; +import { useAggregatedIndicators, UseAggregatedIndicatorsParam } from './use_aggregated_indicators'; import { DEFAULT_TIME_RANGE } from '../../query_bar/hooks/use_filters/utils'; import { - TestProvidersComponent, - mockedSearchService, mockedTimefilterService, + TestProvidersComponent, } from '../../../common/mocks/test_providers'; import { useFilters } from '../../query_bar/hooks/use_filters'; +import { createFetchAggregatedIndicators } from '../services/fetch_aggregated_indicators'; -jest.mock('../../query_bar/hooks/use_filters/use_filters'); - -const aggregationResponse = { - rawResponse: { aggregations: { [AGGREGATION_NAME]: { buckets: [] } } }, -}; - -const calculateBoundsResponse: TimeRangeBounds = { - min: moment('1 Jan 2022 06:00:00 GMT'), - max: moment('1 Jan 2022 12:00:00 GMT'), -}; +jest.mock('../services/fetch_aggregated_indicators'); +jest.mock('../../query_bar/hooks/use_filters'); const useAggregatedIndicatorsParams: UseAggregatedIndicatorsParam = { timeRange: DEFAULT_TIME_RANGE, @@ -40,108 +24,78 @@ const useAggregatedIndicatorsParams: UseAggregatedIndicatorsParam = { const stub = () => {}; +const renderUseAggregatedIndicators = () => + renderHook((props) => useAggregatedIndicators(props), { + initialProps: useAggregatedIndicatorsParams, + wrapper: TestProvidersComponent, + }); + +const initialFiltersValue = { + filters: [], + filterQuery: { language: 'kuery', query: '' }, + filterManager: {} as any, + handleSavedQuery: stub, + handleSubmitQuery: stub, + handleSubmitTimeRange: stub, +}; + describe('useAggregatedIndicators()', () => { beforeEach(jest.clearAllMocks); - beforeEach(() => { - mockedSearchService.search.mockReturnValue(new BehaviorSubject(aggregationResponse)); - mockedTimefilterService.timefilter.calculateBounds.mockReturnValue(calculateBoundsResponse); - }); + type MockedCreateFetchAggregatedIndicators = jest.MockedFunction< + typeof createFetchAggregatedIndicators + >; + let aggregatedIndicatorsQuery: jest.MockedFunction< + ReturnType + >; - describe('when mounted', () => { - beforeEach(() => { - (useFilters as jest.MockedFunction).mockReturnValue({ - filters: [], - filterQuery: { language: 'kuery', query: '' }, - filterManager: {} as any, - handleSavedQuery: stub, - handleSubmitQuery: stub, - handleSubmitTimeRange: stub, - }); - - renderHook(() => useAggregatedIndicators(useAggregatedIndicatorsParams), { - wrapper: TestProvidersComponent, - }); - }); + beforeEach(jest.clearAllMocks); - it('should query the database for threat indicators', async () => { - expect(mockedSearchService.search).toHaveBeenCalledTimes(1); - }); + beforeEach(() => { + aggregatedIndicatorsQuery = jest.fn(); + (createFetchAggregatedIndicators as MockedCreateFetchAggregatedIndicators).mockReturnValue( + aggregatedIndicatorsQuery + ); - it('should use the calculateBounds to convert TimeRange to TimeRangeBounds', () => { - expect(mockedTimefilterService.timefilter.calculateBounds).toHaveBeenCalledTimes(1); - }); + (useFilters as jest.MockedFunction).mockReturnValue(initialFiltersValue); }); - describe('when query fails', () => { - beforeEach(async () => { - mockedSearchService.search.mockReturnValue(throwError(() => new Error('some random error'))); - mockedTimefilterService.timefilter.calculateBounds.mockReturnValue(calculateBoundsResponse); - }); + it('should create and call the aggregatedIndicatorsQuery correctly', async () => { + aggregatedIndicatorsQuery.mockResolvedValue([]); - beforeEach(() => { - renderHook(() => useAggregatedIndicators(useAggregatedIndicatorsParams), { - wrapper: TestProvidersComponent, - }); - }); + const { rerender } = renderUseAggregatedIndicators(); - it('should show an error', async () => { - expect(mockedSearchService.showError).toHaveBeenCalledTimes(1); - - expect(mockedSearchService.search).toHaveBeenCalledWith( - expect.objectContaining({ - params: expect.objectContaining({ - body: expect.objectContaining({ - aggregations: expect.any(Object), - query: expect.any(Object), - size: expect.any(Number), - fields: expect.any(Array), - }), - }), - }), - expect.objectContaining({ - abortSignal: expect.any(AbortSignal), - }) - ); - }); - }); + // indicators service and the query should be called just once + expect( + createFetchAggregatedIndicators as MockedCreateFetchAggregatedIndicators + ).toHaveBeenCalledTimes(1); + expect(aggregatedIndicatorsQuery).toHaveBeenCalledTimes(1); - describe('when query is successful', () => { - beforeEach(async () => { - mockedSearchService.search.mockReturnValue( - new BehaviorSubject>({ - rawResponse: { - aggregations: { - [AGGREGATION_NAME]: { - buckets: [ - { - doc_count: 1, - key: '[Filebeat] AbuseCH Malware', - events: { - buckets: [ - { - doc_count: 0, - key: 1641016800000, - key_as_string: '1 Jan 2022 06:00:00 GMT', - }, - ], - }, - }, - ], - }, - }, - }, - }) - ); - mockedTimefilterService.timefilter.calculateBounds.mockReturnValue(calculateBoundsResponse); + // Ensure the timefilter service is called + expect(mockedTimefilterService.timefilter.calculateBounds).toHaveBeenCalled(); + // Call the query function + expect(aggregatedIndicatorsQuery).toHaveBeenLastCalledWith( + expect.objectContaining({ + filterQuery: { language: 'kuery', query: '' }, + }), + expect.any(AbortSignal) + ); + + // After filter values change, the hook will be re-rendered and should call the query function again, with + // updated values + (useFilters as jest.MockedFunction).mockReturnValue({ + ...initialFiltersValue, + filterQuery: { language: 'kuery', query: "threat.indicator.type: 'file'" }, }); - it('should call mapping function on every hit', async () => { - const { result } = renderHook(() => useAggregatedIndicators(useAggregatedIndicatorsParams), { - wrapper: TestProvidersComponent, - }); + await act(async () => rerender()); - expect(result.current.indicators.length).toEqual(1); - }); + expect(aggregatedIndicatorsQuery).toHaveBeenCalledTimes(2); + expect(aggregatedIndicatorsQuery).toHaveBeenLastCalledWith( + expect.objectContaining({ + filterQuery: { language: 'kuery', query: "threat.indicator.type: 'file'" }, + }), + expect.any(AbortSignal) + ); }); }); diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_aggregated_indicators.ts b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_aggregated_indicators.ts index ab583fb0ed95..2609dbda5eb1 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_aggregated_indicators.ts +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_aggregated_indicators.ts @@ -6,25 +6,20 @@ */ import { TimeRange } from '@kbn/es-query'; -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { Subscription } from 'rxjs'; -import { - IEsSearchRequest, - IKibanaSearchResponse, - isCompleteResponse, - isErrorResponse, - TimeRangeBounds, -} from '@kbn/data-plugin/common'; +import { useMemo, useState } from 'react'; +import { TimeRangeBounds } from '@kbn/data-plugin/common'; +import { useQuery } from '@tanstack/react-query'; import { useInspector } from '../../../hooks/use_inspector'; import { useFilters } from '../../query_bar/hooks/use_filters'; -import { convertAggregationToChartSeries } from '../../../common/utils/barchart'; import { RawIndicatorFieldId } from '../../../../common/types/indicator'; -import { calculateBarchartColumnTimeInterval } from '../../../common/utils/dates'; import { useKibana } from '../../../hooks/use_kibana'; import { DEFAULT_TIME_RANGE } from '../../query_bar/hooks/use_filters/utils'; import { useSourcererDataView } from './use_sourcerer_data_view'; -import { getRuntimeMappings } from '../utils/get_runtime_mappings'; -import { getIndicatorsQuery } from '../utils/get_indicators_query'; +import { + ChartSeries, + createFetchAggregatedIndicators, + FetchAggregatedIndicatorsParams, +} from '../services/fetch_aggregated_indicators'; export interface UseAggregatedIndicatorsParam { /** @@ -55,37 +50,7 @@ export interface UseAggregatedIndicatorsValue { selectedField: string; } -export interface Aggregation { - doc_count: number; - key: string; - events: { - buckets: AggregationValue[]; - }; -} - -export interface AggregationValue { - doc_count: number; - key: number; - key_as_string: string; -} - -export interface ChartSeries { - x: string; - y: number; - g: string; -} - -const TIMESTAMP_FIELD = RawIndicatorFieldId.TimeStamp; const DEFAULT_FIELD = RawIndicatorFieldId.Feed; -export const AGGREGATION_NAME = 'barchartAggregation'; - -export interface RawAggregatedIndicatorsResponse { - aggregations: { - [AGGREGATION_NAME]: { - buckets: Aggregation[]; - }; - }; -} export const useAggregatedIndicators = ({ timeRange = DEFAULT_TIME_RANGE, @@ -100,132 +65,48 @@ export const useAggregatedIndicators = ({ const { inspectorAdapters } = useInspector(); - const searchSubscription$ = useRef(new Subscription()); - const abortController = useRef(new AbortController()); - - const [indicators, setIndicators] = useState([]); const [field, setField] = useState(DEFAULT_FIELD); const { filters, filterQuery } = useFilters(); - const dateRange: TimeRangeBounds = useMemo( - () => queryService.timefilter.timefilter.calculateBounds(timeRange), - [queryService, timeRange] + const aggregatedIndicatorsQuery = useMemo( + () => + createFetchAggregatedIndicators({ + queryService, + searchService, + inspectorAdapter: inspectorAdapters.requests, + }), + [inspectorAdapters, queryService, searchService] ); - const queryToExecute = useMemo(() => { - return getIndicatorsQuery({ timeRange, filters, filterQuery }); - }, [filterQuery, filters, timeRange]); - - const loadData = useCallback(async () => { - const dateFrom: number = (dateRange.min as moment.Moment).toDate().getTime(); - const dateTo: number = (dateRange.max as moment.Moment).toDate().getTime(); - const interval = calculateBarchartColumnTimeInterval(dateFrom, dateTo); - - const request = inspectorAdapters.requests.start('Indicator barchart', {}); - - request.stats({ - indexPattern: { - label: 'Index patterns', - value: selectedPatterns, + const { data } = useQuery( + [ + 'indicatorsBarchart', + { + filters, + field, + filterQuery, + selectedPatterns, + timeRange, }, - }); - - abortController.current = new AbortController(); - - const requestBody = { - aggregations: { - [AGGREGATION_NAME]: { - terms: { - field, - }, - aggs: { - events: { - date_histogram: { - field: TIMESTAMP_FIELD, - fixed_interval: interval, - min_doc_count: 0, - extended_bounds: { - min: dateFrom, - max: dateTo, - }, - }, - }, - }, - }, - }, - fields: [TIMESTAMP_FIELD, field], - size: 0, - query: queryToExecute, - runtime_mappings: getRuntimeMappings(), - }; - - searchSubscription$.current = searchService - .search>( - { - params: { - index: selectedPatterns, - body: requestBody, - }, - }, - { - abortSignal: abortController.current.signal, - } - ) - .subscribe({ - next: (response) => { - if (isCompleteResponse(response)) { - const aggregations: Aggregation[] = - response.rawResponse.aggregations[AGGREGATION_NAME]?.buckets; - const chartSeries: ChartSeries[] = convertAggregationToChartSeries(aggregations); - setIndicators(chartSeries); - searchSubscription$.current.unsubscribe(); - - request.stats({}).ok({ json: response }); - request.json(requestBody); - } else if (isErrorResponse(response)) { - request.error({ json: response }); - searchSubscription$.current.unsubscribe(); - } - }, - error: (requestError) => { - searchService.showError(requestError); - searchSubscription$.current.unsubscribe(); - - if (requestError instanceof Error && requestError.name.includes('Abort')) { - inspectorAdapters.requests.reset(); - } else { - request.error({ json: requestError }); - } - }, - }); - }, [ - dateRange.max, - dateRange.min, - field, - inspectorAdapters.requests, - queryToExecute, - searchService, - selectedPatterns, - ]); - - const onFieldChange = useCallback( - async (f: string) => { - setField(f); - loadData(); - }, - [loadData, setField] + ], + ({ + signal, + queryKey: [_key, queryParams], + }: { + signal?: AbortSignal; + queryKey: [string, FetchAggregatedIndicatorsParams]; + }) => aggregatedIndicatorsQuery(queryParams, signal) ); - useEffect(() => { - loadData(); - - return () => abortController.current.abort(); - }, [loadData]); + const dateRange = useMemo( + () => queryService.timefilter.timefilter.calculateBounds(timeRange), + [queryService.timefilter.timefilter, timeRange] + ); return { dateRange, - indicators, - onFieldChange, + indicators: data || [], + onFieldChange: setField, selectedField: field, }; }; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators.test.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators.test.tsx index 1b1762eb8b67..0b0620813078 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators.test.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators.test.tsx @@ -6,17 +6,11 @@ */ import { renderHook, act } from '@testing-library/react-hooks'; -import { - useIndicators, - RawIndicatorsResponse, - UseIndicatorsParams, - UseIndicatorsValue, -} from './use_indicators'; -import { BehaviorSubject, throwError } from 'rxjs'; -import { TestProvidersComponent, mockedSearchService } from '../../../common/mocks/test_providers'; -import { IKibanaSearchResponse } from '@kbn/data-plugin/public'; - -const indicatorsResponse = { rawResponse: { hits: { hits: [], total: 0 } } }; +import { useIndicators, UseIndicatorsParams, UseIndicatorsValue } from './use_indicators'; +import { TestProvidersComponent } from '../../../common/mocks/test_providers'; +import { createFetchIndicators } from '../services/fetch_indicators'; + +jest.mock('../services/fetch_indicators'); const useIndicatorsParams: UseIndicatorsParams = { filters: [], @@ -24,66 +18,71 @@ const useIndicatorsParams: UseIndicatorsParams = { sorting: [], }; +const indicatorsQueryResult = { indicators: [], total: 0 }; + +const renderUseIndicators = (initialProps = useIndicatorsParams) => + renderHook((props) => useIndicators(props), { + initialProps, + wrapper: TestProvidersComponent, + }); + describe('useIndicators()', () => { + type MockedCreateFetchIndicators = jest.MockedFunction; + let indicatorsQuery: jest.MockedFunction>; + beforeEach(jest.clearAllMocks); + beforeEach(() => { + indicatorsQuery = jest.fn(); + (createFetchIndicators as MockedCreateFetchIndicators).mockReturnValue(indicatorsQuery); + }); + describe('when mounted', () => { - beforeEach(() => { - mockedSearchService.search.mockReturnValue(new BehaviorSubject(indicatorsResponse)); - }); + it('should create and call the indicatorsQuery', async () => { + indicatorsQuery.mockResolvedValue(indicatorsQueryResult); - beforeEach(async () => { - renderHook( - () => useIndicators(useIndicatorsParams), - { - wrapper: TestProvidersComponent, - } - ); - }); + const hookResult = renderUseIndicators(); - it('should query the database for threat indicators', async () => { - expect(mockedSearchService.search).toHaveBeenCalledTimes(1); - }); - }); + // isLoading should be true + expect(hookResult.result.current.isLoading).toEqual(true); + + // indicators service and the query should be called just once + expect(createFetchIndicators as MockedCreateFetchIndicators).toHaveBeenCalledTimes(1); + expect(indicatorsQuery).toHaveBeenCalledTimes(1); - describe('when filters change', () => { - beforeEach(() => { - mockedSearchService.search.mockReturnValue(new BehaviorSubject(indicatorsResponse)); + // isLoading should turn to false eventually + await hookResult.waitFor(() => !hookResult.result.current.isLoading); + expect(hookResult.result.current.isLoading).toEqual(false); }); + }); + describe('when inputs change', () => { it('should query the database again and reset page to 0', async () => { - const hookResult = renderHook( - (props) => useIndicators(props), - { - initialProps: useIndicatorsParams, - wrapper: TestProvidersComponent, - } - ); + const hookResult = renderUseIndicators(); + expect(indicatorsQuery).toHaveBeenCalledTimes(1); + + // Change page + await act(async () => hookResult.result.current.onChangePage(42)); - expect(mockedSearchService.search).toHaveBeenCalledTimes(1); - expect(mockedSearchService.search).toHaveBeenLastCalledWith( + expect(indicatorsQuery).toHaveBeenCalledTimes(2); + expect(indicatorsQuery).toHaveBeenLastCalledWith( expect.objectContaining({ - params: expect.objectContaining({ body: expect.objectContaining({ from: 0 }) }), + pagination: expect.objectContaining({ pageIndex: 42 }), }), - expect.objectContaining({ - abortSignal: expect.any(AbortSignal), - }) + expect.any(AbortSignal) ); - // Change page - await act(async () => hookResult.result.current.onChangePage(42)); + // Change page size + await act(async () => hookResult.result.current.onChangeItemsPerPage(50)); - expect(mockedSearchService.search).toHaveBeenLastCalledWith( + expect(indicatorsQuery).toHaveBeenCalledTimes(3); + expect(indicatorsQuery).toHaveBeenLastCalledWith( expect.objectContaining({ - params: expect.objectContaining({ body: expect.objectContaining({ from: 42 * 25 }) }), + pagination: expect.objectContaining({ pageIndex: 0, pageSize: 50 }), }), - expect.objectContaining({ - abortSignal: expect.any(AbortSignal), - }) + expect.any(AbortSignal) ); - expect(mockedSearchService.search).toHaveBeenCalledTimes(2); - // Change filters act(() => hookResult.rerender({ @@ -92,164 +91,13 @@ describe('useIndicators()', () => { }) ); - // From range should be reset to 0 - expect(mockedSearchService.search).toHaveBeenCalledTimes(3); - expect(mockedSearchService.search).toHaveBeenLastCalledWith( + expect(indicatorsQuery).toHaveBeenLastCalledWith( expect.objectContaining({ - params: expect.objectContaining({ body: expect.objectContaining({ from: 0 }) }), - }), - expect.objectContaining({ - abortSignal: expect.any(AbortSignal), - }) - ); - }); - }); - - describe('when query fails', () => { - beforeEach(async () => { - mockedSearchService.search.mockReturnValue(throwError(() => new Error('some random error'))); - - renderHook((props) => useIndicators(props), { - initialProps: useIndicatorsParams, - wrapper: TestProvidersComponent, - }); - }); - - it('should show an error', async () => { - expect(mockedSearchService.showError).toHaveBeenCalledTimes(1); - - expect(mockedSearchService.search).toHaveBeenCalledWith( - expect.objectContaining({ - params: expect.objectContaining({ - body: expect.objectContaining({ - query: expect.any(Object), - from: expect.any(Number), - size: expect.any(Number), - fields: expect.any(Array), - }), - }), + pagination: expect.objectContaining({ pageIndex: 0 }), + filterQuery: { language: 'kuery', query: "threat.indicator.type: 'file'" }, }), - expect.objectContaining({ - abortSignal: expect.any(AbortSignal), - }) + expect.any(AbortSignal) ); }); }); - - describe('when query is successful', () => { - beforeEach(async () => { - mockedSearchService.search.mockReturnValue( - new BehaviorSubject>({ - rawResponse: { hits: { hits: [{ fields: {} }], total: 1 } }, - }) - ); - }); - - it('should call mapping function on every hit', async () => { - const { result } = renderHook( - (props) => useIndicators(props), - { - initialProps: useIndicatorsParams, - wrapper: TestProvidersComponent, - } - ); - expect(result.current.indicatorCount).toEqual(1); - }); - }); - - describe('pagination', () => { - beforeEach(async () => { - mockedSearchService.search.mockReturnValue( - new BehaviorSubject>({ - rawResponse: { hits: { hits: [{ fields: {} }], total: 1 } }, - }) - ); - }); - - describe('when page changes', () => { - it('should run the query again with pagination parameters', async () => { - const { result } = renderHook( - () => useIndicators(useIndicatorsParams), - { - wrapper: TestProvidersComponent, - } - ); - - await act(async () => { - result.current.onChangePage(42); - }); - - expect(mockedSearchService.search).toHaveBeenCalledTimes(2); - - expect(mockedSearchService.search).toHaveBeenCalledWith( - expect.objectContaining({ - params: expect.objectContaining({ - body: expect.objectContaining({ - size: 25, - from: 0, - }), - }), - }), - expect.anything() - ); - - expect(mockedSearchService.search).toHaveBeenLastCalledWith( - expect.objectContaining({ - params: expect.objectContaining({ - body: expect.objectContaining({ - size: 25, - from: 42 * 25, - }), - }), - }), - expect.anything() - ); - - expect(result.current.pagination.pageIndex).toEqual(42); - }); - - describe('when page size changes', () => { - it('should fetch the first page and update internal page size', async () => { - const { result } = renderHook( - () => useIndicators(useIndicatorsParams), - { - wrapper: TestProvidersComponent, - } - ); - - await act(async () => { - result.current.onChangeItemsPerPage(50); - }); - - expect(mockedSearchService.search).toHaveBeenCalledTimes(3); - - expect(mockedSearchService.search).toHaveBeenCalledWith( - expect.objectContaining({ - params: expect.objectContaining({ - body: expect.objectContaining({ - size: 25, - from: 0, - }), - }), - }), - expect.anything() - ); - - expect(mockedSearchService.search).toHaveBeenLastCalledWith( - expect.objectContaining({ - params: expect.objectContaining({ - body: expect.objectContaining({ - size: 50, - from: 0, - }), - }), - }), - expect.anything() - ); - - expect(result.current.pagination.pageIndex).toEqual(0); - }); - }); - }); - }); }); diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators.ts b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators.ts index f06fb03b27d5..5303b5dae06c 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators.ts +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators.ts @@ -5,21 +5,14 @@ * 2.0. */ -import { - IEsSearchRequest, - IKibanaSearchResponse, - isCompleteResponse, - isErrorResponse, -} from '@kbn/data-plugin/common'; -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import type { Subscription } from 'rxjs'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { Filter, Query, TimeRange } from '@kbn/es-query'; +import { useQuery } from '@tanstack/react-query'; import { useInspector } from '../../../hooks/use_inspector'; import { Indicator } from '../../../../common/types/indicator'; import { useKibana } from '../../../hooks/use_kibana'; import { useSourcererDataView } from './use_sourcerer_data_view'; -import { getRuntimeMappings } from '../utils/get_runtime_mappings'; -import { getIndicatorsQuery } from '../utils/get_indicators_query'; +import { createFetchIndicators, FetchParams, Pagination } from '../services/fetch_indicators'; const PAGE_SIZES = [10, 25, 50]; @@ -39,20 +32,16 @@ export interface UseIndicatorsValue { pagination: Pagination; onChangeItemsPerPage: (value: number) => void; onChangePage: (value: number) => void; - loading: boolean; -} -export interface RawIndicatorsResponse { - hits: { - hits: any[]; - total: number; - }; -} + /** + * No data loaded yet + */ + isLoading: boolean; -export interface Pagination { - pageSize: number; - pageIndex: number; - pageSizeOptions: number[]; + /** + * Data loading is in progress (see docs on `isFetching` here: https://tanstack.com/query/v4/docs/guides/queries) + */ + isFetching: boolean; } export const useIndicators = ({ @@ -70,12 +59,20 @@ export const useIndicators = ({ const { inspectorAdapters } = useInspector(); - const searchSubscription$ = useRef(); - const abortController = useRef(new AbortController()); + const onChangeItemsPerPage = useCallback( + (pageSize) => + setPagination((currentPagination) => ({ + ...currentPagination, + pageSize, + pageIndex: 0, + })), + [] + ); - const [indicators, setIndicators] = useState([]); - const [indicatorCount, setIndicatorCount] = useState(0); - const [loading, setLoading] = useState(true); + const onChangePage = useCallback( + (pageIndex) => setPagination((currentPagination) => ({ ...currentPagination, pageIndex })), + [] + ); const [pagination, setPagination] = useState({ pageIndex: 0, @@ -83,119 +80,52 @@ export const useIndicators = ({ pageSizeOptions: PAGE_SIZES, }); - const query = useMemo( - () => getIndicatorsQuery({ filters, timeRange, filterQuery }), - [filterQuery, filters, timeRange] - ); - - const loadData = useCallback( - async (from: number, size: number) => { - abortController.current = new AbortController(); - - setLoading(true); - - const request = inspectorAdapters.requests.start('Indicator search', {}); - - request.stats({ - indexPattern: { - label: 'Index patterns', - value: selectedPatterns, - }, - }); - - const requestBody = { - query, - runtime_mappings: getRuntimeMappings(), - fields: [{ field: '*', include_unmapped: true }], - size, - from, - sort: sorting.map(({ id, direction }) => ({ [id]: direction })), - }; - - searchSubscription$.current = searchService - .search>( - { - params: { - index: selectedPatterns, - body: requestBody, - }, - }, - { - abortSignal: abortController.current.signal, - } - ) - .subscribe({ - next: (response) => { - setIndicators(response.rawResponse.hits.hits); - setIndicatorCount(response.rawResponse.hits.total || 0); - - if (isCompleteResponse(response)) { - setLoading(false); - searchSubscription$.current?.unsubscribe(); - request.stats({}).ok({ json: response }); - request.json(requestBody); - } else if (isErrorResponse(response)) { - setLoading(false); - request.error({ json: response }); - searchSubscription$.current?.unsubscribe(); - } - }, - error: (requestError) => { - searchService.showError(requestError); - searchSubscription$.current?.unsubscribe(); - - if (requestError instanceof Error && requestError.name.includes('Abort')) { - inspectorAdapters.requests.reset(); - } else { - request.error({ json: requestError }); - } - - setLoading(false); - }, - }); - }, - [inspectorAdapters.requests, query, searchService, selectedPatterns, sorting] - ); - - const onChangeItemsPerPage = useCallback( - async (pageSize) => { - setPagination((currentPagination) => ({ - ...currentPagination, - pageSize, - pageIndex: 0, - })); + // Go to first page after filters are changed + useEffect(() => { + onChangePage(0); + }, [filters, filterQuery, timeRange, sorting, onChangePage]); - loadData(0, pageSize); - }, - [loadData] + const fetchIndicators = useMemo( + () => createFetchIndicators({ searchService, inspectorAdapter: inspectorAdapters.requests }), + [inspectorAdapters, searchService] ); - const onChangePage = useCallback( - async (pageIndex) => { - setPagination((currentPagination) => ({ ...currentPagination, pageIndex })); - loadData(pageIndex * pagination.pageSize, pagination.pageSize); - }, - [loadData, pagination.pageSize] + const { isLoading, isFetching, data, refetch } = useQuery( + [ + 'indicatorsTable', + { + timeRange, + filterQuery, + filters, + selectedPatterns, + sorting, + pagination, + }, + ], + ({ signal, queryKey: [_key, queryParams] }) => + fetchIndicators(queryParams as FetchParams, signal), + { + /** + * See https://tanstack.com/query/v4/docs/guides/paginated-queries + * This is essential for our ux + */ + keepPreviousData: true, + } ); const handleRefresh = useCallback(() => { onChangePage(0); - }, [onChangePage]); - - // Initial data load (on mount) - useEffect(() => { - handleRefresh(); - - return () => abortController.current.abort(); - }, [handleRefresh]); + refetch(); + }, [onChangePage, refetch]); return { - indicators, - indicatorCount, + indicators: data?.indicators || [], + indicatorCount: data?.total || 0, pagination, onChangePage, onChangeItemsPerPage, - loading, + isLoading, + isFetching, handleRefresh, }; }; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators_total_count.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators_total_count.tsx index 9c49ad8126a2..d99d7c0fc4b0 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators_total_count.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/hooks/use_indicators_total_count.tsx @@ -12,8 +12,8 @@ import { isCompleteResponse, } from '@kbn/data-plugin/common'; import { useKibana } from '../../../hooks/use_kibana'; -import type { RawIndicatorsResponse } from './use_indicators'; import { useSourcererDataView } from './use_sourcerer_data_view'; +import type { RawIndicatorsResponse } from '../services/fetch_indicators'; export const useIndicatorsTotalCount = () => { const { diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/indicators_page.test.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/indicators_page.test.tsx index 371f917c2774..11bdb8fc8e6e 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/indicators_page.test.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/indicators_page.test.tsx @@ -35,7 +35,8 @@ describe('', () => { (useIndicators as jest.MockedFunction).mockReturnValue({ indicators: [{ fields: {} }], indicatorCount: 1, - loading: false, + isLoading: false, + isFetching: false, pagination: { pageIndex: 0, pageSize: 10, pageSizeOptions: [10] }, onChangeItemsPerPage: stub, onChangePage: stub, diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/indicators_page.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/indicators_page.tsx index 8c138ffec502..f51e062e1c3c 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/indicators_page.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/indicators_page.tsx @@ -6,6 +6,7 @@ */ import React, { FC, VFC } from 'react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { IndicatorsFilters } from './containers/indicators_filters/indicators_filters'; import { IndicatorsBarChartWrapper } from './components/indicators_barchart_wrapper/indicators_barchart_wrapper'; import { IndicatorsTable } from './components/indicators_table/indicators_table'; @@ -19,10 +20,14 @@ import { FieldTypesProvider } from '../../containers/field_types_provider'; import { InspectorProvider } from '../../containers/inspector'; import { useColumnSettings } from './components/indicators_table/hooks/use_column_settings'; +const queryClient = new QueryClient(); + const IndicatorsPageProviders: FC = ({ children }) => ( - - {children} - + + + {children} + + ); const IndicatorsPageContent: VFC = () => { @@ -49,36 +54,35 @@ const IndicatorsPageContent: VFC = () => { }); return ( - - - - - - - - - - - + + + + + + + + + ); }; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_aggregated_indicators.test.ts b/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_aggregated_indicators.test.ts new file mode 100644 index 000000000000..c5503f1b32a0 --- /dev/null +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_aggregated_indicators.test.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mockedQueryService, mockedSearchService } from '../../../common/mocks/test_providers'; +import { BehaviorSubject, throwError } from 'rxjs'; +import { RequestAdapter } from '@kbn/inspector-plugin/common'; +import { AGGREGATION_NAME, createFetchAggregatedIndicators } from './fetch_aggregated_indicators'; + +const aggregationResponse = { + rawResponse: { aggregations: { [AGGREGATION_NAME]: { buckets: [] } } }, +}; + +describe('FetchAggregatedIndicatorsService', () => { + beforeEach(jest.clearAllMocks); + + describe('aggregatedIndicatorsQuery()', () => { + describe('when query is successful', () => { + beforeEach(() => { + mockedSearchService.search.mockReturnValue(new BehaviorSubject(aggregationResponse)); + }); + + it('should pass the query down to searchService', async () => { + const aggregatedIndicatorsQuery = createFetchAggregatedIndicators({ + searchService: mockedSearchService, + queryService: mockedQueryService as any, + inspectorAdapter: new RequestAdapter(), + }); + + const result = await aggregatedIndicatorsQuery({ + selectedPatterns: [], + filterQuery: { language: 'kuery', query: '' }, + filters: [], + field: 'myField', + timeRange: { + from: '', + to: '', + }, + }); + + expect(mockedSearchService.search).toHaveBeenCalled(); + expect(mockedSearchService.search).toHaveBeenCalledWith( + expect.objectContaining({ + params: expect.objectContaining({ + body: expect.objectContaining({ + size: 0, + query: expect.objectContaining({ bool: expect.anything() }), + runtime_mappings: { + 'threat.indicator.name': { script: expect.anything(), type: 'keyword' }, + 'threat.indicator.name_origin': { script: expect.anything(), type: 'keyword' }, + }, + aggregations: { + [AGGREGATION_NAME]: { + terms: { + field: 'myField', + }, + aggs: { + events: { + date_histogram: { + field: '@timestamp', + fixed_interval: expect.anything(), + min_doc_count: 0, + extended_bounds: expect.anything(), + }, + }, + }, + }, + }, + fields: ['@timestamp', 'myField'], + }), + index: [], + }), + }), + expect.anything() + ); + + expect(result).toMatchInlineSnapshot(`Array []`); + }); + }); + + describe('when query fails', () => { + beforeEach(() => { + mockedSearchService.search.mockReturnValue( + throwError(() => new Error('some random exception')) + ); + }); + + it('should throw an error', async () => { + const aggregatedIndicatorsQuery = createFetchAggregatedIndicators({ + searchService: mockedSearchService, + queryService: mockedQueryService as any, + inspectorAdapter: new RequestAdapter(), + }); + + try { + await aggregatedIndicatorsQuery({ + selectedPatterns: [], + filterQuery: { language: 'kuery', query: '' }, + filters: [], + field: 'myField', + timeRange: { + from: '', + to: '', + }, + }); + } catch (error) { + expect(error).toMatchInlineSnapshot(`[Error: some random exception]`); + } + + expect.assertions(1); + }); + }); + }); +}); diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_aggregated_indicators.ts b/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_aggregated_indicators.ts new file mode 100644 index 000000000000..6cf0fea18b2c --- /dev/null +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_aggregated_indicators.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TimeRangeBounds } from '@kbn/data-plugin/common'; +import type { ISearchStart, QueryStart } from '@kbn/data-plugin/public'; +import type { Filter, Query, TimeRange } from '@kbn/es-query'; +import { RequestAdapter } from '@kbn/inspector-plugin/common'; +import { convertAggregationToChartSeries } from '../../../common/utils/barchart'; +import { calculateBarchartColumnTimeInterval } from '../../../common/utils/dates'; +import { RawIndicatorFieldId } from '../../../../common/types/indicator'; +import { getIndicatorQueryParams } from '../utils/get_indicator_query_params'; +import { search } from '../utils/search'; + +const TIMESTAMP_FIELD = RawIndicatorFieldId.TimeStamp; + +export const AGGREGATION_NAME = 'barchartAggregation'; + +export interface AggregationValue { + doc_count: number; + key: number; + key_as_string: string; +} + +export interface Aggregation { + doc_count: number; + key: string; + events: { + buckets: AggregationValue[]; + }; +} + +export interface RawAggregatedIndicatorsResponse { + aggregations: { + [AGGREGATION_NAME]: { + buckets: Aggregation[]; + }; + }; +} + +export interface ChartSeries { + x: string; + y: number; + g: string; +} + +export interface FetchAggregatedIndicatorsParams { + selectedPatterns: string[]; + filters: Filter[]; + filterQuery: Query; + timeRange: TimeRange; + field: string; +} + +export const createFetchAggregatedIndicators = + ({ + inspectorAdapter, + searchService, + queryService, + }: { + inspectorAdapter: RequestAdapter; + searchService: ISearchStart; + queryService: QueryStart; + }) => + async ( + { selectedPatterns, timeRange, field, filterQuery, filters }: FetchAggregatedIndicatorsParams, + signal?: AbortSignal + ): Promise => { + const dateRange: TimeRangeBounds = + queryService.timefilter.timefilter.calculateBounds(timeRange); + + const dateFrom: number = (dateRange.min as moment.Moment).toDate().getTime(); + const dateTo: number = (dateRange.max as moment.Moment).toDate().getTime(); + const interval = calculateBarchartColumnTimeInterval(dateFrom, dateTo); + + const sharedParams = getIndicatorQueryParams({ timeRange, filters, filterQuery }); + + const searchRequestBody = { + aggregations: { + [AGGREGATION_NAME]: { + terms: { + field, + }, + aggs: { + events: { + date_histogram: { + field: TIMESTAMP_FIELD, + fixed_interval: interval, + min_doc_count: 0, + extended_bounds: { + min: dateFrom, + max: dateTo, + }, + }, + }, + }, + }, + }, + fields: [TIMESTAMP_FIELD, field], + size: 0, + ...sharedParams, + }; + + const { + aggregations: { [AGGREGATION_NAME]: aggregation }, + } = await search( + searchService, + { + params: { + index: selectedPatterns, + body: searchRequestBody, + }, + }, + { signal, inspectorAdapter, requestName: 'Indicators barchart' } + ); + + const aggregations: Aggregation[] = aggregation?.buckets; + + const chartSeries: ChartSeries[] = convertAggregationToChartSeries(aggregations); + + return chartSeries; + }; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_indicators.test.ts b/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_indicators.test.ts new file mode 100644 index 000000000000..388b1c9c9e7c --- /dev/null +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_indicators.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mockedSearchService } from '../../../common/mocks/test_providers'; +import { BehaviorSubject, throwError } from 'rxjs'; +import { createFetchIndicators } from './fetch_indicators'; +import { RequestAdapter } from '@kbn/inspector-plugin/common'; + +const indicatorsResponse = { rawResponse: { hits: { hits: [], total: 0 } } }; + +describe('FetchIndicatorsService', () => { + beforeEach(jest.clearAllMocks); + + describe('indicatorsQuery()', () => { + describe('when query is successful', () => { + beforeEach(() => { + mockedSearchService.search.mockReturnValue(new BehaviorSubject(indicatorsResponse)); + }); + + it('should pass the query down to searchService', async () => { + const indicatorsQuery = createFetchIndicators({ + searchService: mockedSearchService, + inspectorAdapter: new RequestAdapter(), + }); + + const result = await indicatorsQuery({ + pagination: { + pageIndex: 0, + pageSize: 25, + pageSizeOptions: [1, 2, 3], + }, + selectedPatterns: [], + sorting: [], + filterQuery: { language: 'kuery', query: '' }, + filters: [], + }); + + expect(mockedSearchService.search).toHaveBeenCalled(); + expect(mockedSearchService.search).toHaveBeenCalledWith( + expect.objectContaining({ + params: { + body: { + fields: [{ field: '*', include_unmapped: true }], + from: 0, + query: expect.objectContaining({ bool: expect.anything() }), + runtime_mappings: { + 'threat.indicator.name': { script: expect.anything(), type: 'keyword' }, + 'threat.indicator.name_origin': { script: expect.anything(), type: 'keyword' }, + }, + size: 25, + sort: [], + }, + index: [], + }, + }), + expect.anything() + ); + + expect(result).toMatchInlineSnapshot(` + Object { + "indicators": Array [], + "total": 0, + } + `); + }); + }); + + describe('when query fails', () => { + beforeEach(() => { + mockedSearchService.search.mockReturnValue( + throwError(() => new Error('some random exception')) + ); + }); + + it('should throw an error', async () => { + const indicatorsQuery = createFetchIndicators({ + searchService: mockedSearchService, + inspectorAdapter: new RequestAdapter(), + }); + + try { + await indicatorsQuery({ + pagination: { + pageIndex: 0, + pageSize: 25, + pageSizeOptions: [1, 2, 3], + }, + selectedPatterns: [], + sorting: [], + filterQuery: { language: 'kuery', query: '' }, + filters: [], + }); + } catch (error) { + expect(error).toMatchInlineSnapshot(`[Error: some random exception]`); + } + + expect.assertions(1); + }); + }); + }); +}); diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_indicators.ts b/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_indicators.ts new file mode 100644 index 000000000000..f06038c32011 --- /dev/null +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/services/fetch_indicators.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ISearchStart } from '@kbn/data-plugin/public'; +import type { Filter, Query, TimeRange } from '@kbn/es-query'; +import { RequestAdapter } from '@kbn/inspector-plugin/common'; +import { Indicator } from '../../../../common/types/indicator'; +import { getIndicatorQueryParams } from '../utils/get_indicator_query_params'; +import { search } from '../utils/search'; + +export interface RawIndicatorsResponse { + hits: { + hits: any[]; + total: number; + }; +} + +export interface Pagination { + pageSize: number; + pageIndex: number; + pageSizeOptions: number[]; +} + +interface FetchIndicatorsDependencies { + searchService: ISearchStart; + inspectorAdapter: RequestAdapter; +} + +export interface FetchParams { + pagination: Pagination; + selectedPatterns: string[]; + sorting: any[]; + filters: Filter[]; + timeRange?: TimeRange; + filterQuery: Query; +} + +type ReactQueryKey = [string, FetchParams]; + +export interface IndicatorsQueryParams { + signal?: AbortSignal; + queryKey: ReactQueryKey; +} + +export interface IndicatorsResponse { + indicators: Indicator[]; + total: number; +} + +export const createFetchIndicators = + ({ searchService, inspectorAdapter }: FetchIndicatorsDependencies) => + async ( + { pagination, selectedPatterns, timeRange, filterQuery, filters, sorting }: FetchParams, + signal?: AbortSignal + ): Promise => { + const sharedParams = getIndicatorQueryParams({ timeRange, filters, filterQuery }); + + const searchRequestBody = { + size: pagination.pageSize, + from: pagination.pageIndex, + fields: [{ field: '*', include_unmapped: true } as const], + sort: sorting.map(({ id, direction }) => ({ [id]: direction })), + ...sharedParams, + }; + + const { + hits: { hits: indicators, total }, + } = await search( + searchService, + { + params: { + index: selectedPatterns, + body: searchRequestBody, + }, + }, + { inspectorAdapter, requestName: 'Indicators table', signal } + ); + + return { indicators, total }; + }; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/utils/get_indicator_query_params.ts b/x-pack/plugins/threat_intelligence/public/modules/indicators/utils/get_indicator_query_params.ts new file mode 100644 index 000000000000..bcaf304fdfd7 --- /dev/null +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/utils/get_indicator_query_params.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildEsQuery, Filter, Query, TimeRange } from '@kbn/es-query'; +import { THREAT_QUERY_BASE } from '../../../../common/constants'; +import { RawIndicatorFieldId } from '../../../../common/types/indicator'; +import { threatIndicatorNamesOriginScript, threatIndicatorNamesScript } from './display_name'; + +const TIMESTAMP_FIELD = RawIndicatorFieldId.TimeStamp; + +/** + * Prepare shared `runtime_mappings` and `query` fields used within indicator search request + */ +export const getIndicatorQueryParams = ({ + filters, + filterQuery, + timeRange, +}: { + filters: Filter[]; + filterQuery: Query; + timeRange?: TimeRange; +}) => { + return { + runtime_mappings: { + [RawIndicatorFieldId.Name]: { + type: 'keyword', + script: { + source: threatIndicatorNamesScript(), + }, + }, + [RawIndicatorFieldId.NameOrigin]: { + type: 'keyword', + script: { + source: threatIndicatorNamesOriginScript(), + }, + }, + } as const, + query: buildEsQuery( + undefined, + [ + { + query: THREAT_QUERY_BASE, + language: 'kuery', + }, + { + query: filterQuery.query as string, + language: 'kuery', + }, + ], + [ + ...filters, + { + query: { + range: { + [TIMESTAMP_FIELD]: { + gte: timeRange?.from, + lte: timeRange?.to, + }, + }, + }, + meta: {}, + }, + ] + ), + }; +}; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/utils/get_indicators_query.ts b/x-pack/plugins/threat_intelligence/public/modules/indicators/utils/get_indicators_query.ts deleted file mode 100644 index 160fa22db763..000000000000 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/utils/get_indicators_query.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { buildEsQuery, Filter, Query, TimeRange } from '@kbn/es-query'; -import { THREAT_QUERY_BASE } from '../../../../common/constants'; -import { RawIndicatorFieldId } from '../../../../common/types/indicator'; - -const TIMESTAMP_FIELD = RawIndicatorFieldId.TimeStamp; - -export const getIndicatorsQuery = ({ - filters, - filterQuery, - timeRange, -}: { - filters: Filter[]; - filterQuery: Query; - timeRange?: TimeRange; -}) => { - return buildEsQuery( - undefined, - [ - { - query: THREAT_QUERY_BASE, - language: 'kuery', - }, - { - query: filterQuery.query as string, - language: 'kuery', - }, - ], - [ - ...filters, - { - query: { - range: { - [TIMESTAMP_FIELD]: { - gte: timeRange?.from, - lte: timeRange?.to, - }, - }, - }, - meta: {}, - }, - ] - ); -}; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/utils/search.ts b/x-pack/plugins/threat_intelligence/public/modules/indicators/utils/search.ts new file mode 100644 index 000000000000..49cd371680ce --- /dev/null +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/utils/search.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + IEsSearchRequest, + IKibanaSearchResponse, + isCompleteResponse, + isErrorResponse, +} from '@kbn/data-plugin/common'; +import { ISearchStart } from '@kbn/data-plugin/public'; +import { RequestAdapter } from '@kbn/inspector-plugin/common'; + +interface SearchOptions { + /** + * Inspector adapter, available in the context + */ + inspectorAdapter: RequestAdapter; + + /** + * Request name registered in the inspector panel + */ + requestName: string; + + /** + * Abort signal + */ + signal?: AbortSignal; +} + +/** + * This is a searchService wrapper that will instrument your query with `inspector` and turn it into a Promise, + * resolved when complete result set is returned or rejected on any error, other than Abort. + */ +export const search = async ( + searchService: ISearchStart, + searchRequest: IEsSearchRequest, + { inspectorAdapter, requestName, signal }: SearchOptions +): Promise => { + const requestId = `${Date.now()}`; + const request = inspectorAdapter.start(requestName, { id: requestId }); + + return new Promise((resolve, reject) => { + searchService + .search>(searchRequest, { + abortSignal: signal, + }) + .subscribe({ + next: (response) => { + if (isCompleteResponse(response)) { + request.stats({}).ok({ json: response }); + request.json(searchRequest.params?.body || {}); + + resolve(response.rawResponse); + } else if (isErrorResponse(response)) { + request.error({ json: response }); + reject(response); + } + }, + error: (requestError) => { + if (requestError instanceof Error && requestError.name.includes('Abort')) { + inspectorAdapter.resetRequest(requestId); + } else { + request.error({ json: requestError }); + } + + searchService.showError(requestError); + reject(requestError); + }, + }); + }); +}; diff --git a/x-pack/plugins/threat_intelligence/public/modules/query_bar/components/query_bar/query_bar.tsx b/x-pack/plugins/threat_intelligence/public/modules/query_bar/components/query_bar/query_bar.tsx index d2739476d7d6..e564c898f894 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/query_bar/components/query_bar/query_bar.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/query_bar/components/query_bar/query_bar.tsx @@ -82,15 +82,17 @@ export const QueryBar = memo( }) => { const onQuerySubmit = useCallback( ({ query, dateRange }: QueryPayload) => { - if (isQuery(query) && !deepEqual(query, filterQuery)) { - onSubmitQuery(query); - } - if (dateRange != null) { onSubmitDateRange(dateRange); } + + if (isQuery(query) && !deepEqual(query, filterQuery)) { + onSubmitQuery(query); + } else { + onRefresh(); + } }, - [filterQuery, onSubmitDateRange, onSubmitQuery] + [filterQuery, onRefresh, onSubmitDateRange, onSubmitQuery] ); const onQueryChange = useCallback( @@ -169,7 +171,6 @@ export const QueryBar = memo( dataTestSubj={dataTestSubj} savedQuery={savedQuery} displayStyle={displayStyle} - onRefresh={onRefresh} /> ); } diff --git a/x-pack/plugins/threat_intelligence/scripts/generate_indicators.js b/x-pack/plugins/threat_intelligence/scripts/generate_indicators.js index 44e0ad166353..0d5c170965fe 100644 --- a/x-pack/plugins/threat_intelligence/scripts/generate_indicators.js +++ b/x-pack/plugins/threat_intelligence/scripts/generate_indicators.js @@ -50,7 +50,7 @@ const main = async () => { 'threat.feed.name': { type: 'keyword', }, - 'threat.indicator.url.original': { + 'threat.indicator.url.full': { type: 'keyword', }, 'threat.indicator.first_seen': { @@ -92,7 +92,7 @@ const main = async () => { 'threat.indicator.first_seen': timestamp, 'threat.feed.name': FEED_NAMES[Math.ceil(Math.random() * FEED_NAMES.length) - 1], 'threat.indicator.type': 'url', - 'threat.indicator.url.original': faker.internet.url(), + 'threat.indicator.url.full': faker.internet.url(), 'event.type': 'indicator', 'event.category': 'threat', }, From cc9827e396b166fbdad536381420272d614c0b98 Mon Sep 17 00:00:00 2001 From: Katerina Patticha Date: Tue, 27 Sep 2022 15:20:19 +0200 Subject: [PATCH 2/4] [APM] Generate Service metrics with synthrace (#141739) --- .../src/cli/run_synthtrace.ts | 4 +- .../src/cli/utils/synthtrace_worker.ts | 4 +- ...gator.ts => service_metrics_aggregator.ts} | 68 ++++++++++++++----- 3 files changed, 55 insertions(+), 21 deletions(-) rename packages/kbn-apm-synthtrace/src/lib/apm/aggregators/{service_latency_aggregator.ts => service_metrics_aggregator.ts} (78%) diff --git a/packages/kbn-apm-synthtrace/src/cli/run_synthtrace.ts b/packages/kbn-apm-synthtrace/src/cli/run_synthtrace.ts index ecfbd4a387f3..517d2d61d799 100644 --- a/packages/kbn-apm-synthtrace/src/cli/run_synthtrace.ts +++ b/packages/kbn-apm-synthtrace/src/cli/run_synthtrace.ts @@ -15,7 +15,7 @@ import { parseRunCliFlags } from './utils/parse_run_cli_flags'; import { getCommonServices } from './utils/get_common_services'; import { ApmSynthtraceKibanaClient } from '../lib/apm/client/apm_synthtrace_kibana_client'; import { StreamAggregator } from '../lib/stream_aggregator'; -import { ServiceLatencyAggregator } from '../lib/apm/aggregators/service_latency_aggregator'; +import { ServicMetricsAggregator } from '../lib/apm/aggregators/service_metrics_aggregator'; function options(y: Argv) { return y @@ -207,7 +207,7 @@ export function runSynthtrace() { } const aggregators: StreamAggregator[] = []; const registry = new Map StreamAggregator[]>([ - ['service', () => [new ServiceLatencyAggregator()]], + ['service', () => [new ServicMetricsAggregator()]], ]); if (runOptions.streamProcessors && runOptions.streamProcessors.length > 0) { for (const processorName of runOptions.streamProcessors) { diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts index 720b1b0527e8..54ce5b1b2e32 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts @@ -16,7 +16,7 @@ import { StreamProcessor } from '../../lib/stream_processor'; import { Scenario } from '../scenario'; import { EntityIterable, Fields } from '../../..'; import { StreamAggregator } from '../../lib/stream_aggregator'; -import { ServiceLatencyAggregator } from '../../lib/apm/aggregators/service_latency_aggregator'; +import { ServicMetricsAggregator } from '../../lib/apm/aggregators/service_metrics_aggregator'; // logging proxy to main thread, ensures we see real time logging const l = { @@ -63,7 +63,7 @@ async function setup() { parentPort?.postMessage({ workerIndex, lastTimestamp: item['@timestamp'] }); } }; - const aggregators: StreamAggregator[] = [new ServiceLatencyAggregator()]; + const aggregators: StreamAggregator[] = [new ServicMetricsAggregator()]; // If we are sending data to apm-server we do not have to create any aggregates in the stream processor streamProcessor = new StreamProcessor({ version, diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/aggregators/service_latency_aggregator.ts b/packages/kbn-apm-synthtrace/src/lib/apm/aggregators/service_metrics_aggregator.ts similarity index 78% rename from packages/kbn-apm-synthtrace/src/lib/apm/aggregators/service_latency_aggregator.ts rename to packages/kbn-apm-synthtrace/src/lib/apm/aggregators/service_metrics_aggregator.ts index e28ba234b2a4..618c9e52b9f2 100644 --- a/packages/kbn-apm-synthtrace/src/lib/apm/aggregators/service_latency_aggregator.ts +++ b/packages/kbn-apm-synthtrace/src/lib/apm/aggregators/service_metrics_aggregator.ts @@ -12,12 +12,14 @@ import { ApmFields } from '../apm_fields'; import { Fields } from '../../entity'; import { StreamAggregator } from '../../stream_aggregator'; -type LatencyState = { +type AggregationState = { count: number; min: number; max: number; sum: number; timestamp: number; + failure_count: number; + success_count: number; } & Pick; export type ServiceFields = Fields & @@ -35,15 +37,22 @@ export type ServiceFields = Fields & | 'transaction.type' > & Partial<{ - 'transaction.duration.aggregate': { - min: number; - max: number; - sum: number; - value_count: number; + _doc_count: number; + transaction: { + duration: { + summary: { + min: number; + max: number; + sum: number; + value_count: number; + }; + }; + failure_count: number; + success_count: number; }; }>; -export class ServiceLatencyAggregator implements StreamAggregator { +export class ServicMetricsAggregator implements StreamAggregator { public readonly name; constructor() { @@ -68,7 +77,7 @@ export class ServiceLatencyAggregator implements StreamAggregator { duration: { type: 'object', properties: { - aggregate: { + summary: { type: 'aggregate_metric_double', metrics: ['min', 'max', 'sum', 'value_count'], default_metric: 'sum', @@ -76,6 +85,12 @@ export class ServiceLatencyAggregator implements StreamAggregator { }, }, }, + failure_count: { + type: { type: 'long' }, + }, + success_count: { + type: { type: 'long' }, + }, }, }, service: { @@ -99,7 +114,7 @@ export class ServiceLatencyAggregator implements StreamAggregator { return null; } - private state: Record = {}; + private state: Record = {}; private processedComponent: number = 0; @@ -120,13 +135,25 @@ export class ServiceLatencyAggregator implements StreamAggregator { 'service.name': service, 'service.environment': environment, 'transaction.type': transactionType, + failure_count: 0, + success_count: 0, }; } + + const state = this.state[key]; + state.count++; + + switch (event['event.outcome']) { + case 'failure': + state.failure_count++; + break; + case 'success': + state.success_count++; + break; + } + const duration = Number(event['transaction.duration.us']); if (duration >= 0) { - const state = this.state[key]; - - state.count++; state.sum += duration; if (duration > state.max) state.max = duration; if (duration < state.min) state.min = Math.min(0, duration); @@ -164,17 +191,24 @@ export class ServiceLatencyAggregator implements StreamAggregator { const component = Date.now() % 100; const state = this.state[key]; return { + _doc_count: state.count, '@timestamp': state.timestamp + random(0, 100) + component + this.processedComponent, 'metricset.name': 'service', 'processor.event': 'metric', 'service.name': state['service.name'], 'service.environment': state['service.environment'], 'transaction.type': state['transaction.type'], - 'transaction.duration.aggregate': { - min: state.min, - max: state.max, - sum: state.sum, - value_count: state.count, + transaction: { + duration: { + summary: { + min: state.min, + max: state.max, + sum: state.sum, + value_count: state.count, + }, + }, + failure_count: state.failure_count, + success_count: state.success_count, }, }; } From f2937926ffd79714bf18af587d4b4b6c8fc14b6f Mon Sep 17 00:00:00 2001 From: Jan Monschke Date: Tue, 27 Sep 2022 16:27:43 +0300 Subject: [PATCH 3/4] fix: don't show process ancestry insights in some cases (#141751) --- .../event_details/insights/insights.test.tsx | 33 +++++++++++++++++-- .../event_details/insights/insights.tsx | 21 ++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/insights/insights.test.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/insights/insights.test.tsx index 747b2ec939f7..b31cf1d1252b 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/insights/insights.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/insights/insights.test.tsx @@ -61,7 +61,7 @@ jest.mock('../../../hooks/use_experimental_features', () => ({ })); const useIsExperimentalFeatureEnabledMock = useIsExperimentalFeatureEnabled as jest.Mock; -const data: TimelineEventsDetailsItem[] = [ +const dataWithoutAgentType: TimelineEventsDetailsItem[] = [ { category: 'process', field: 'process.entity_id', @@ -82,6 +82,16 @@ const data: TimelineEventsDetailsItem[] = [ }, ]; +const data: TimelineEventsDetailsItem[] = [ + ...dataWithoutAgentType, + { + category: 'agent', + field: 'agent.type', + isObjectArray: false, + values: ['endpoint'], + }, +]; + describe('Insights', () => { beforeEach(() => { mockUseGetUserCasesPermissions.mockReturnValue(noCasesPermissions()); @@ -123,9 +133,11 @@ describe('Insights', () => { describe('with feature flag enabled', () => { describe('with platinum license', () => { - it('should show insights for related alerts by process ancestry', () => { + beforeAll(() => { licenseServiceMock.isPlatinumPlus.mockReturnValue(true); + }); + it('should show insights for related alerts by process ancestry', () => { render( @@ -137,6 +149,23 @@ describe('Insights', () => { screen.queryByRole('link', { name: new RegExp(i18n.ALERT_UPSELL) }) ).not.toBeInTheDocument(); }); + + describe('without process ancestry info', () => { + it('should not show the related alerts by process ancestry insights module', () => { + render( + + + + ); + + expect(screen.queryByTestId('related-alerts-by-ancestry')).not.toBeInTheDocument(); + }); + }); }); describe('without platinum license', () => { diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/insights/insights.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/insights/insights.tsx index 358b2d8833a6..520f30fb9c31 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/insights/insights.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/insights/insights.tsx @@ -40,7 +40,6 @@ export const Insights = React.memo( 'insightsRelatedAlertsByProcessAncestry' ); const hasAtLeastPlatinum = useLicense().isPlatinumPlus(); - const processEntityField = find({ category: 'process', field: 'process.entity_id' }, data); const originalDocumentId = find( { category: 'kibana', field: 'kibana.alert.ancestors.id' }, data @@ -49,7 +48,12 @@ export const Insights = React.memo( { category: 'kibana', field: 'kibana.alert.rule.parameters.index' }, data ); - const hasProcessEntityInfo = hasData(processEntityField); + const agentTypeField = find({ category: 'agent', field: 'agent.type' }, data); + const eventModuleField = find({ category: 'event', field: 'event.module' }, data); + const processEntityField = find({ category: 'process', field: 'process.entity_id' }, data); + const hasProcessEntityInfo = + hasData(processEntityField) && + hasCorrectAgentTypeAndEventModule(agentTypeField, eventModuleField); const processSessionField = find( { category: 'process', field: 'process.entry_leader.entity_id' }, @@ -147,4 +151,17 @@ export const Insights = React.memo( } ); +function hasCorrectAgentTypeAndEventModule( + agentTypeField?: TimelineEventsDetailsItem, + eventModuleField?: TimelineEventsDetailsItem +): boolean { + return ( + hasData(agentTypeField) && + (agentTypeField.values[0] === 'endpoint' || + (agentTypeField.values[0] === 'winlogbeat' && + hasData(eventModuleField) && + eventModuleField.values[0] === 'sysmon')) + ); +} + Insights.displayName = 'Insights'; From fb136431c5a47fe8bb9fab4672d88a80f1d8b760 Mon Sep 17 00:00:00 2001 From: Michael Olorunnisola Date: Tue, 27 Sep 2022 09:30:17 -0400 Subject: [PATCH 4/4] [Security Solution][Timeline][Fix] - Use dynamic language for search bar (#137606) * use search bar language in place of hard coded kuery * update tests * update snapshot * add cypress --- .../e2e/timelines/search_or_filter.cy.ts | 15 ++++++++++++++- .../cypress/screens/timeline.ts | 10 ++++++++++ .../cypress/tasks/timeline.ts | 19 +++++++++++++++++++ .../__snapshots__/index.test.tsx.snap | 1 + .../timeline/query_tab_content/index.test.tsx | 1 + .../timeline/query_tab_content/index.tsx | 11 +++++++++-- .../timeline/epic_local_storage.test.tsx | 1 + 7 files changed, 55 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/e2e/timelines/search_or_filter.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/timelines/search_or_filter.cy.ts index d5a084f65fac..39420b27e386 100644 --- a/x-pack/plugins/security_solution/cypress/e2e/timelines/search_or_filter.cy.ts +++ b/x-pack/plugins/security_solution/cypress/e2e/timelines/search_or_filter.cy.ts @@ -16,7 +16,11 @@ import { cleanKibana } from '../../tasks/common'; import { login, visit, visitWithoutDateRange } from '../../tasks/login'; import { openTimelineUsingToggle } from '../../tasks/security_main'; -import { executeTimelineKQL } from '../../tasks/timeline'; +import { + changeTimelineQueryLanguage, + executeTimelineKQL, + executeTimelineSearch, +} from '../../tasks/timeline'; import { waitForTimelinesPanelToBeLoaded } from '../../tasks/timelines'; import { HOSTS_URL, TIMELINES_URL } from '../../urls/navigation'; @@ -38,6 +42,15 @@ describe('Timeline search and filters', () => { cy.get(SERVER_SIDE_EVENT_COUNT).should(($count) => expect(+$count.text()).to.be.gt(0)); }); + + it('executes a Lucene query', () => { + const messageProcessQuery = 'message:Process\\ zsh*'; + openTimelineUsingToggle(); + changeTimelineQueryLanguage('lucene'); + executeTimelineSearch(messageProcessQuery); + + cy.get(SERVER_SIDE_EVENT_COUNT).should(($count) => expect(+$count.text()).to.be.gt(0)); + }); }); describe('Update kqlMode for timeline', () => { diff --git a/x-pack/plugins/security_solution/cypress/screens/timeline.ts b/x-pack/plugins/security_solution/cypress/screens/timeline.ts index 51ce3b38d4d6..87d70a73dbd1 100644 --- a/x-pack/plugins/security_solution/cypress/screens/timeline.ts +++ b/x-pack/plugins/security_solution/cypress/screens/timeline.ts @@ -215,6 +215,16 @@ export const TIMELINE_KQLMODE_SEARCH = '[data-test-subj="kqlModePopoverSearch"]' export const TIMELINE_KQLMODE_FILTER = '[data-test-subj="kqlModePopoverFilter"]'; +export const QUERYBAR_MENU_POPOVER = '[data-test-subj="queryBarMenuPopover"]'; + +export const TIMELINE_SHOWQUERYBARMENU_BUTTON = `${TIMELINE_FLYOUT} [data-test-subj="showQueryBarMenu"]`; + +export const TIMELINE_SWITCHQUERYLANGUAGE_BUTTON = '[data-test-subj="switchQueryLanguageButton"]'; + +export const TIMELINE_LUCENELANGUAGE_BUTTON = '[data-test-subj="luceneLanguageMenuItem"]'; + +export const TIMELINE_KQLLANGUAGE_BUTTON = '[data-test-subj="kqlLanguageMenuItem"]'; + export const TIMELINE_TITLE = '[data-test-subj="timeline-title"]'; export const TIMELINE_TITLE_INPUT = '[data-test-subj="save-timeline-title"]'; diff --git a/x-pack/plugins/security_solution/cypress/tasks/timeline.ts b/x-pack/plugins/security_solution/cypress/tasks/timeline.ts index fda2ea08769e..a83e1157e98d 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/timeline.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/timeline.ts @@ -74,6 +74,11 @@ import { EMPTY_DROPPABLE_DATA_PROVIDER_GROUP, GET_TIMELINE_GRID_CELL, HOVER_ACTIONS, + TIMELINE_SWITCHQUERYLANGUAGE_BUTTON, + TIMELINE_SHOWQUERYBARMENU_BUTTON, + TIMELINE_LUCENELANGUAGE_BUTTON, + TIMELINE_KQLLANGUAGE_BUTTON, + TIMELINE_QUERY, } from '../screens/timeline'; import { REFRESH_BUTTON, TIMELINE } from '../screens/timelines'; import { drag, drop } from './common'; @@ -170,6 +175,16 @@ export const addFilter = (filter: TimelineFilter): Cypress.Chainable { + cy.get(TIMELINE_SHOWQUERYBARMENU_BUTTON).click(); + cy.get(TIMELINE_SWITCHQUERYLANGUAGE_BUTTON).click(); + if (language === 'lucene') { + cy.get(TIMELINE_LUCENELANGUAGE_BUTTON).click(); + } else { + cy.get(TIMELINE_KQLLANGUAGE_BUTTON).click(); + } +}; + export const addDataProvider = (filter: TimelineFilter): Cypress.Chainable> => { cy.get(TIMELINE_ADD_FIELD_BUTTON).click(); cy.get(LOADING_INDICATOR).should('not.exist'); @@ -280,6 +295,10 @@ export const executeTimelineKQL = (query: string) => { cy.get(`${SEARCH_OR_FILTER_CONTAINER} textarea`).type(`${query} {enter}`); }; +export const executeTimelineSearch = (query: string) => { + cy.get(TIMELINE_QUERY).type(`${query} {enter}`, { force: true }); +}; + export const expandFirstTimelineEventDetails = () => { cy.get(TOGGLE_TIMELINE_EXPAND_EVENT).first().click({ force: true }); }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/__snapshots__/index.test.tsx.snap index 5b0757ee775f..3205a30d35c0 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/__snapshots__/index.test.tsx.snap @@ -305,6 +305,7 @@ In other use cases the message field can be used to concatenate different values } kqlMode="search" kqlQueryExpression=" " + kqlQueryLanguage="kuery" onEventClosed={[MockFunction]} renderCellValue={[Function]} rowRenderers={ diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.test.tsx index b610adbe6da5..cc48a58717ee 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.test.tsx @@ -137,6 +137,7 @@ describe('Timeline', () => { itemsPerPageOptions: [5, 10, 20], kqlMode: 'search' as QueryTabContentComponentProps['kqlMode'], kqlQueryExpression: ' ', + kqlQueryLanguage: 'kuery', onEventClosed: jest.fn(), renderCellValue: DefaultCellRenderer, rowRenderers: defaultRowRenderers, diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.tsx index c38304d79841..414604109e58 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.tsx @@ -179,6 +179,7 @@ export const QueryTabContentComponent: React.FC = ({ itemsPerPageOptions, kqlMode, kqlQueryExpression, + kqlQueryLanguage, onEventClosed, renderCellValue, rowRenderers, @@ -223,8 +224,8 @@ export const QueryTabContentComponent: React.FC = ({ query: string; language: KueryFilterQueryKind; } = useMemo( - () => ({ query: kqlQueryExpression.trim(), language: 'kuery' }), - [kqlQueryExpression] + () => ({ query: kqlQueryExpression.trim(), language: kqlQueryLanguage }), + [kqlQueryExpression, kqlQueryLanguage] ); const combinedQueries = combineQueries({ @@ -493,6 +494,11 @@ const makeMapStateToProps = () => { ? ' ' : kqlQueryTimeline?.expression ?? ''; + const kqlQueryLanguage = + isEmpty(dataProviders) && timelineType === 'template' + ? 'kuery' + : kqlQueryTimeline?.kind ?? 'kuery'; + return { activeTab, columns, @@ -506,6 +512,7 @@ const makeMapStateToProps = () => { itemsPerPageOptions, kqlMode, kqlQueryExpression, + kqlQueryLanguage, showCallOutUnauthorizedMsg: getShowCallOutUnauthorizedMsg(state), show, showExpandedDetails: diff --git a/x-pack/plugins/security_solution/public/timelines/store/timeline/epic_local_storage.test.tsx b/x-pack/plugins/security_solution/public/timelines/store/timeline/epic_local_storage.test.tsx index 42a7460a129d..e68530e4579c 100644 --- a/x-pack/plugins/security_solution/public/timelines/store/timeline/epic_local_storage.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/store/timeline/epic_local_storage.test.tsx @@ -78,6 +78,7 @@ describe('epicLocalStorage', () => { itemsPerPageOptions: [5, 10, 20], kqlMode: 'search' as QueryTabContentComponentProps['kqlMode'], kqlQueryExpression: '', + kqlQueryLanguage: 'kuery', onEventClosed: jest.fn(), renderCellValue: DefaultCellRenderer, rowRenderers: defaultRowRenderers,