Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[2.x] Add log pattern table (#1187) #1212

Merged
merged 2 commits into from
Nov 1, 2022
Merged

[2.x] Add log pattern table (#1187) #1212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bf9306f
Pick pr/1187 to 2.x
joshuali925 Oct 28, 2022
ebf8d1b
Update snapshots
joshuali925 Nov 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dashboards-observability/common/constants/explorer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export const RAW_QUERY = 'rawQuery';
export const FINAL_QUERY = 'finalQuery';
export const SELECTED_DATE_RANGE = 'selectedDateRange';
export const INDEX = 'index';
export const SELECTED_PATTERN = 'selectedPattern';
export const SELECTED_TIMESTAMP = 'selectedTimestamp';
export const SELECTED_FIELDS = 'selectedFields';
export const UNSELECTED_FIELDS = 'unselectedFields';
Expand Down Expand Up @@ -74,6 +75,8 @@ export const REDUX_EXPL_SLICE_FIELDS = 'fields';
export const REDUX_EXPL_SLICE_QUERY_TABS = 'queryTabs';
export const REDUX_EXPL_SLICE_VISUALIZATION = 'explorerVisualization';
export const REDUX_EXPL_SLICE_COUNT_DISTRIBUTION = 'countDistributionVisualization';
export const REDUX_EXPL_SLICE_PATTERNS = 'patterns';
export const PLOTLY_GAUGE_COLUMN_NUMBER = 5;
export const APP_ANALYTICS_TAB_ID_REGEX = /application-analytics-tab.+/;
export const DEFAULT_AVAILABILITY_QUERY = 'stats count() by span( timestamp, 1h )';
export const PPL_PATTERNS_REGEX = /\|\s*patterns\s+\S+\s*\|\s*where\s+patterns_field\s*\=\s*'[^a-zA-Z0-9]+'/;
12 changes: 9 additions & 3 deletions dashboards-observability/common/types/explorer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,8 @@ export interface SavedQuery {
name: string;
query: string;
selected_date_range: { start: string; end: string; text: string };
selected_fields: { text: string; tokens: [{ name: string; type: string }] };
selected_timestamp: { name: string; type: string };
selected_fields: { text: string; tokens: IField[] };
selected_timestamp: IField;
}

export interface SavedVisualization {
Expand All @@ -129,7 +129,7 @@ export interface SavedVisualization {
query: string;
selected_date_range: { start: string; end: string; text: string };
selected_fields: { text: string; tokens: [] };
selected_timestamp: { name: string; type: string };
selected_timestamp: IField;
type: string;
application_id?: string;
}
Expand Down Expand Up @@ -227,3 +227,9 @@ export interface LiveTailProps {
isLiveTailPopoverOpen: boolean;
dataTestSubj: string;
}

export interface PatternTableData {
count: number;
pattern: string;
sampleLog: string;
}
9 changes: 8 additions & 1 deletion dashboards-observability/public/components/application_analytics/helpers/utils.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { FilterType } from 'public/components/trace_analytics/components/common/
import React, { Dispatch, ReactChild } from 'react';
import { batch } from 'react-redux';
import PPLService from 'public/services/requests/ppl';
import { IField } from '../../../../common/types/explorer';
import { preprocessQuery } from '../../../../common/utils/query_utils';
import { SPAN_REGEX } from '../../../../common/constants/shared';
import { fetchVisualizationById } from '../../../components/custom_panels/helpers/utils';
Expand All @@ -36,6 +37,10 @@ import {
remove as removeQueryResult,
} from '../../event_analytics/redux/slices/query_result_slice';
import { addTab, removeTab } from '../../event_analytics/redux/slices/query_tab_slice';
import {
init as initPatterns,
remove as removePatterns,
} from '../../event_analytics/redux/slices/patterns_slice';

// Name validation
export const isNameValid = (name: string, existingNames: string[]) => {
Expand Down Expand Up @@ -153,6 +158,7 @@ export const removeTabData = (
[NEW_SELECTED_QUERY_TAB]: newIdToFocus,
})
);
dispatch(removePatterns({ tabId: TabIdToBeClosed }));
});
};

Expand All @@ -172,6 +178,7 @@ export const initializeTabData = async (dispatch: Dispatch<any>, tabId: string,
},
})
);
dispatch(initPatterns({ tabId }));
});
};

Expand Down Expand Up @@ -234,7 +241,7 @@ export const calculateAvailability = async (
})
.then((res) => {
const stat = res.metadata.fields.filter(
(field: { name: string; type: string }) => !field.name.match(SPAN_REGEX)
(field: IField) => !field.name.match(SPAN_REGEX)
)[0].name;
const value = res.data[stat];
currValue = value[value.length - 1];
Expand Down
160 changes: 146 additions & 14 deletions dashboards-observability/public/components/event_analytics/explorer/explorer.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import React, { useState, useMemo, useEffect, useRef, useCallback, ReactElement
import { batch, useDispatch, useSelector } from 'react-redux';
import { isEmpty, cloneDeep, isEqual, has, reduce } from 'lodash';
import { FormattedMessage } from '@osd/i18n/react';
import { EuiLoadingSpinner, EuiSpacer } from '@elastic/eui';
import { EuiHorizontalRule, EuiLoadingSpinner, EuiSpacer, EuiTitle } from '@elastic/eui';
import {
EuiText,
EuiButtonIcon,
Expand Down Expand Up @@ -51,6 +51,8 @@ import {
TAB_CHART_ID,
DEFAULT_AVAILABILITY_QUERY,
DATE_PICKER_FORMAT,
PPL_PATTERNS_REGEX,
SELECTED_PATTERN,
} from '../../../../common/constants/explorer';
import {
PPL_STATS_REGEX,
Expand All @@ -77,6 +79,9 @@ import { getVizContainerProps } from '../../visualizations/charts/helpers';
import { parseGetSuggestions, onItemSelect } from '../../common/search/autocomplete_logic';
import { formatError } from '../utils';
import { sleep } from '../../common/live_tail/live_tail_button';
import { PatternsTable } from './log_patterns/patterns_table';
import { selectPatterns } from '../redux/slices/patterns_slice';
import { useFetchPatterns } from '../hooks/use_fetch_patterns';

const TYPE_TAB_MAPPING = {
[SAVED_QUERY]: TAB_EVENT_ID,
Expand Down Expand Up @@ -116,13 +121,18 @@ export const Explorer = ({
pplService,
requestParams,
});
const { getPatterns, setDefaultPatternsField } = useFetchPatterns({
pplService,
requestParams,
});
const appLogEvents = tabId.startsWith('application-analytics-tab');
const query = useSelector(selectQueries)[tabId];
const explorerData = useSelector(selectQueryResult)[tabId];
const explorerFields = useSelector(selectFields)[tabId];
const countDistribution = useSelector(selectCountDistribution)[tabId];
const explorerVisualizations = useSelector(selectExplorerVisualization)[tabId];
const userVizConfigs = useSelector(selectVisualizationConfig)[tabId] || {};
const patternsData = useSelector(selectPatterns)[tabId];
const [selectedContentTabId, setSelectedContentTab] = useState(TAB_EVENT_ID);
const [selectedCustomPanelOptions, setSelectedCustomPanelOptions] = useState([]);
const [selectedPanelName, setSelectedPanelName] = useState('');
Expand All @@ -132,6 +142,7 @@ export const Explorer = ({
const [isSidebarClosed, setIsSidebarClosed] = useState(false);
const [timeIntervalOptions, setTimeIntervalOptions] = useState(TIME_INTERVAL_OPTIONS);
const [isOverridingTimestamp, setIsOverridingTimestamp] = useState(false);
const [isOverridingPattern, setIsOverridingPattern] = useState(false);
const [tempQuery, setTempQuery] = useState(query[RAW_QUERY]);
const [isLiveTailPopoverOpen, setIsLiveTailPopoverOpen] = useState(false);
const [isLiveTailOn, setIsLiveTailOn] = useState(false);
Expand All @@ -141,7 +152,7 @@ export const Explorer = ({
const [browserTabFocus, setBrowserTabFocus] = useState(true);
const [liveTimestamp, setLiveTimestamp] = useState(DATE_PICKER_FORMAT);
const [triggerAvailability, setTriggerAvailability] = useState(false);

const [viewLogPatterns, setViewLogPatterns] = useState(false);
const queryRef = useRef();
const appBasedRef = useRef('');
appBasedRef.current = appBaseQuery;
Expand Down Expand Up @@ -196,6 +207,15 @@ export const Explorer = ({
});
});

const getErrorHandler = (title: string) => {
return (error: any) => {
const formattedError = formatError(error.name, error.message, error.body.message);
notifications.toasts.addError(formattedError, {
title,
});
};
};

const composeFinalQuery = (
curQuery: any,
startingTime: string,
Expand Down Expand Up @@ -327,6 +347,19 @@ export const Explorer = ({
}
}

let curPattern: string = curQuery![SELECTED_PATTERN];

if (isEmpty(curPattern)) {
const patternErrorHandler = getErrorHandler('Error fetching default pattern field');
await setDefaultPatternsField(curIndex, '', patternErrorHandler);
const newQuery = queryRef.current;
curPattern = newQuery![SELECTED_PATTERN];
if (isEmpty(curPattern)) {
setToast('Index does not contain a valid pattern field.', 'danger');
return;
}
}

if (isEqual(typeof startingTime, 'undefined') && isEqual(typeof endingTime, 'undefined')) {
startingTime = curQuery![SELECTED_DATE_RANGE][0];
endingTime = curQuery![SELECTED_DATE_RANGE][1];
Expand Down Expand Up @@ -358,21 +391,17 @@ export const Explorer = ({
} else {
findAutoInterval(startTime, endTime);
if (isLiveTailOnRef.current) {
getLiveTail(undefined, (error) => {
const formattedError = formatError(error.name, error.message, error.body.message);
notifications.toasts.addError(formattedError, {
title: 'Error fetching events',
});
});
getLiveTail(undefined, getErrorHandler('Error fetching events'));
} else {
getEvents(undefined, (error) => {
const formattedError = formatError(error.name, error.message, error.body.message);
notifications.toasts.addError(formattedError, {
title: 'Error fetching events',
});
});
getEvents(undefined, getErrorHandler('Error fetching events'));
}
getCountVisualizations(minInterval);

// to fetch patterns data on current query
if (!finalQuery.match(PPL_PATTERNS_REGEX)) {
const patternErrorHandler = getErrorHandler('Error fetching patterns');
getPatterns(patternErrorHandler);
}
}

// for comparing usage if for the same tab, user changed index from one to another
Expand Down Expand Up @@ -538,6 +567,17 @@ export const Explorer = ({
handleQuerySearch();
};

const handleOverridePattern = async (pattern: IField) => {
setIsOverridingPattern(true);
await setDefaultPatternsField(
'',
pattern.name,
getErrorHandler('Error overriding default pattern')
);
setIsOverridingPattern(false);
await getPatterns(getErrorHandler('Error fetching patterns'));
};

const totalHits: number = useMemo(() => {
if (isLiveTailOn && countDistribution?.data) {
const hits = reduce(
Expand All @@ -553,6 +593,21 @@ export const Explorer = ({
return 0;
}, [countDistribution?.data]);

const onPatternSelection = async (pattern: string) => {
let currQuery = queryRef.current![RAW_QUERY] as string;
const currPattern = queryRef.current![SELECTED_PATTERN] as string;
// Remove existing pattern selection if it exists
if (currQuery.match(PPL_PATTERNS_REGEX)) {
currQuery = currQuery.replace(PPL_PATTERNS_REGEX, '');
}
const patternSelectQuery = `${currQuery.trim()} | patterns ${currPattern} | where patterns_field = '${pattern}'`;
// Passing in empty string will remove pattern query
const newQuery = pattern ? patternSelectQuery : currQuery;
await setTempQuery(newQuery);
await updateQueryInStore(newQuery);
await handleTimeRangePickerRefresh(true);
};

const getMainContent = () => {
return (
<main className="container-fluid">
Expand All @@ -569,10 +624,13 @@ export const Explorer = ({
explorerFields={explorerFields}
explorerData={explorerData}
selectedTimestamp={query[SELECTED_TIMESTAMP]}
selectedPattern={query[SELECTED_PATTERN]}
handleOverrideTimestamp={handleOverrideTimestamp}
handleOverridePattern={handleOverridePattern}
handleAddField={(field: IField) => handleAddField(field)}
handleRemoveField={(field: IField) => handleRemoveField(field)}
isOverridingTimestamp={isOverridingTimestamp}
isOverridingPattern={isOverridingPattern}
isFieldToggleButtonDisabled={
isEmpty(explorerData.jsonData) ||
!isEmpty(queryRef.current![RAW_QUERY].match(PPL_STATS_REGEX))
Expand Down Expand Up @@ -628,6 +686,58 @@ export const Explorer = ({
</EuiFlexItem>
</EuiFlexGroup>
<CountDistribution countDistribution={countDistribution} />
<EuiHorizontalRule margin="xs" />
<EuiFlexGroup
justifyContent="spaceBetween"
alignItems="center"
style={{ margin: '8px' }}
gutterSize="xs"
>
<EuiFlexItem grow={false}>
{viewLogPatterns && (
<EuiTitle size="s">
<h3 style={{ margin: '0px' }}>
Patterns
<span className="pattern-header-count">
{' '}
(
{patternsData.patternTableData
? patternsData.patternTableData.length
: 0}
)
</span>
</h3>
</EuiTitle>
)}
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiFlexGroup>
<EuiFlexItem grow={false}>
{viewLogPatterns && (
<EuiLink onClick={() => onPatternSelection('')}>
Clear Selection
</EuiLink>
)}
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiLink onClick={() => setViewLogPatterns(!viewLogPatterns)}>
{`${viewLogPatterns ? 'Hide' : 'Show'} Patterns`}
</EuiLink>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
</EuiFlexGroup>
<EuiHorizontalRule margin="xs" />
{viewLogPatterns && (
<>
<PatternsTable
tableData={patternsData.patternTableData || []}
onPatternSelection={onPatternSelection}
tabId={tabId}
/>
<EuiHorizontalRule margin="xs" />
</>
)}
</>
)}

Expand Down Expand Up @@ -662,6 +772,26 @@ export const Explorer = ({
<EuiSpacer size="m" />
</>
)}
{countDistribution?.data && (
<EuiTitle size="s">
<h3 style={{ margin: '0px', textAlign: 'left', marginLeft: '10px' }}>
Events
<span className="event-header-count">
{' '}
(
{reduce(
countDistribution.data['count()'],
(sum, n) => {
return sum + n;
},
0
)}
)
</span>
</h3>
</EuiTitle>
)}
<EuiHorizontalRule margin="xs" />
<DataGrid
http={http}
pplService={pplService}
Expand Down Expand Up @@ -775,6 +905,8 @@ export const Explorer = ({
visualizations,
query,
isLiveTailOnRef.current,
patternsData,
viewLogPatterns,
]);

const handleContentTabClick = (selectedTab: IQueryTab) => setSelectedContentTab(selectedTab.id);
Expand Down
Loading