From a0c49af8d0eb9da2bcedca05f0790b39d3a93cf1 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 19 Sep 2019 16:09:36 -0400 Subject: [PATCH] [lens] Show field details on click (#44697) * [lens] Initial Commit (#35627) * [visualization editor] Initial Commit * [lens] Add more complete initial state * [lens] Fix type issues * [lens] Remove feature control * [lens] Bring back feature control and add tests * [lens] Update plugin structure and naming per comments * replace any usage by safe casting * [lens] Respond to review comments * [lens] Remove unused EditorFrameState type * [lens] Initial state for IndexPatternDatasource (#36052) * [lens] Add first tests to indexpattern data source * Respond to review comments * Fix type definitions * [lens] Editor frame initializes datasources and visualizations (#36060) * [lens] Editor frame initializes datasources and visualizations * Respond to review comments * Fix build issues * Fix state management issue * [lens][draft] Lens/drag drop (#36268) Add basic drag / drop component to Lens * remove local package (#36456) * [lens] Native renderer (#36165) * Add nativerenderer component * Use native renderer in app and editor frame * [Lens] No explicit any (#36515) * [Lens] Implement basic editor frame state handling (#36443) * [lens] Load index patterns and render in data panel (#36463) * [lens] Editor frame initializes datasources and visualizations * Respond to review comments * Fix build issues * remove local package * [lens] Load index patterns into data source * Redo types for Index Pattern Datasource * Fix one more type * Respond to review comments * [draft] Lens/line chart renderer (#36827) Expression logic for the Lens xy chart. * [lens] Index pattern data panel (initial) (#37015) * [lens] Index pattern switcher * Respond to review comments * [Lens] Editor state 2 (#36513) * [lens] Dimension panel that generates columns (#37117) * [lens] Dimension panel that generates columns * Update from review comments * [lens] Generate esdocs queries from index pattern (#37361) * [lens] Generate esdocs queries from index pattern * Remove unused code * Update yarn.lock from yarn kbn bootstrap * [Lens] Add basic Lens xy chart suggestions (#37030) Basic xy chart suggestions * [Lens] Expression rendering (#37648) * [Lens] Expression handling (#37876) * [Lens] Lens/xy config panel (#37391) Basic xy chart configuration panel * [Lens] Xy expression building (#37967) * [Lens] Initialize visualization with datasource api (#38142) * [lens] Dimension panel lets users select operations and fields individually (#37573) * [lens] Dimension panel lets users select operations and fields individually * Split files and add tests * Fix dimension labeling and add clear button * Support more aggregations, aggregation nesting, rollups, and clearing * Fix esaggs expression * Increase top-level test coverage of dimension panel * Update from review comments * [Lens] Rename columns (#38278) * [Lens] Lens/index pattern drag drop (#37711) * Basic xy chart suggestions * Re-apply XY config panel after force merge * Initial integration of lens drag and drop * Tweak naming, remove irellevant comment * Tweaks per Wylie's feedback * Add xy chart internationalization Tweak types per Joe's feedback * Update xy chart i18n implementation * Fix i18n id * Add drop tests to the lens index pattern * improve tests * [lens] Only allow aggregated dimensions (#38820) * [lens] Only allow aggregated dimensions * [lens] Index pattern suggest on drop * Fully remove value * Revert "[lens] Index pattern suggest on drop" This reverts commit 604c6ed68ca394441ddafa662bdfc5f421de300c. * Fix type errors * [lens] Suggest on drop (#38848) * [lens] Index pattern suggest on drop * Add test for suggestion without date field * fix merge * [Lens] Parameter configurations and new dimension config flow (#38863) * fix eslint failure * [lens] Fix build by updating saved objects and i18n (#39391) * [lens] Update location of saved objects code * Update internatationalization * Remove added file * [lens] Fix arguments to esaggs using booleans (#39462) * [lens] Datatable visualization plugin (#39390) * [lens] Datatable visualization plugin * Fix merge issues and add tests * Update from review * Fix file locations * [lens] Use first suggestion when switching visualizations (#39377) * [lens] Label each Y axis with its operation label (#39461) * [lens] Label each Y axis with its operation label * Remove comment * Add link to chart issue * [Lens] Suggestion preview rendering (#39576) * [Lens] Popover configs (#39565) * [Lens] Basic layouting (#39587) * remove datasource public API in suggestions (#39772) * [Lens] Basic save / load (#39257) Add basic routing, save, and load to Lens * [lens] Fix lint error * [lens] Use node scripts/eslint.js --fix to fix errors * [lens] Include link to lens from Visualize (#40542) * [lens] Support stacking in xy visualization (#40546) * [lens] Support stacking in xy visualization * Use chart type switcher for stacked and horizontal xy charts * Clean up remaining isStacked code * Fix type error * [Lens] Add xy split series support (#39726) * Add split series to lens xy chart * [lens] Lens Filter Ratio (#40196) * WIP filter ratio * Fix test issues * Pass dependencies through plugin like new platform * Pass props into filter ratio popover editor * Provide mocks to filter_ratio popover test * Add another test * Clean up to prepare for review * Clean up unnecessary changes * Respond to review comments * Fix tests * [Lens] Terms order direction (#39884) * fix types * [Lens] Data panel styling and optimizations (#40787) Style the data panel (mostly Joe Reuter's doing). Optimize a bunch of the Lens stack. * [Lens] Optimize dimension panel flow (#41114) * [Lens] re-introduce no-explicit-any (#41454) * [Lens] No results marker (#41450) * [lens] Support layers for visualizing results of multiple queries (#41290) * [lens] WIP add support for layers * [lens] WIP switch to nested tables * Get basic layering to work * Load multiple tables and render in one chart * Fix priority ordering * Reduce quantity of linting errors * Ensure that new xy layer state has a split column * Make the "add" y / split the trailing accessor * Various fixes for datasource public API and implementation * Unify datasource deletion and accessor removal * Fix broken scss * Fix xy visualization TypeScript errors? * Build basic suggestions * Restore save/load and fix typescript bugs * simplify init routine * fix save tests * fix persistence tests * fix state management tests * Ensure the data table is aligned to the top * Add layer support to Lens datatable * Give xy chart a default layer initially * Allow deletion of layers in xy charts * xy: Make split accessor singular Remove commented code blocks * Change expression type for lens_merge_tables * Fix XY chart rendering expression * Fix type errors relating to `layerId` in table suggestions * Pass around tables for suggestions with associated layerIds * fix tests in workspace panel * fix editor_frame tests * Fix xy tests, skip inapplicable tests that will be implemented in a separate PR * add some tests for multiple datasources and layers * Suggest that split series comes before X axis in XY chart * Get datatable suggestion working * Adjust how xy axis labels are computed * Datasource suggestions should handle layers and have tests * Fix linting in XY chart and remove commented code * Update snapshots from earlier change * Fix linting errors * More cleanup * Remove commented code * Test the multi-column editor * XY Visualization does not need to track datasourceId * Fix various comments * Remove unused xy prop Add datasource header to datatable config * Use operation labels for XY chart * Adding and removing layers is reflected in the datasource * rewrote datasource state init * clean up editor_frame frame api implementation * clean up editor frame * [Lens] Embeddable (#41361) * [lens] Move XY chart config into popover and fix layering (#41927) * [lens] Move XY chart config into popover and fix layering * Fix tests * Update style * Change wrapping of layer settings popover * [Lens] Fix bugs in date_histogram and filter ratio (#42046) * [Lens] Performance improvements (#41784) * fix type error * switch default size of terms operation to 3 (#42334) * [lens] Improve suggestions for split series (#42052) * [lens] Add chart switcher (#42093) * solve merge conflicts * fix test case * [Lens] Allow only current visualization on field drop in workspace (#42344) * [Lens] Remove indexpattern id on column (#42429) * [lens] Implement app-level filtering and time picker (#42031) * [lens] Implement app-level filtering and time picker * More integration with filter bar * Clean up test code and type errors * Add frame level tests for syncing with app * Add test coverage for app logic * Simplify state management from app down * Fix import errors * Clarify whether properties are ids or titles for index pattern * pass new saved object by ref * add dirty state checking * Fix tests * [Lens] Add some tests around document handling in dimension panel (#42670) * [Lens] Terms operation boolean support (#42817) * [lens] Minor UX/UI improvements in Lens (#42852) * Make dimension popover toggle when clicking button * Without suggestions hide suggestion panel * Add missing translations (#42921) * [Lens] Config panel design (#42980) * Fix up design of config panel Does not include config popover * Remove a couple of non-null assertions (#43013) * Remove a couple of non-null assertions * Remove orphaned import * [Lens] Switch indexpattern manually (#42599) * [Lens] Update frame to put suggestions at the bottom (#42997) * fix type errors * switch indexpattern on layer if there is only a single empty one (#43079) * [Lens] Suggest reduced versions of current data table (#42537) * [Lens] Field formatter support (#38874) * Fix bugs * [Lens] Add bucket nesting editor to indexpattern (#42869) * [Lens] Remove unnecessary fields and indexing from mappings (#43285) * [Lens] Xy scale type (#42142) * [lens] Allow updater function to be used for updating state (#43373) * [Lens] Lens metric visualization (#39364) * Fix axis rotation (#43792) * [Lens] Auto date histogram (#43775) * Add auto date histogram * Improve documentation and cleanup * Add tests * Change test name * [Lens] Fix query bar integration (#43865) * [Lens] Clean up operations code (#43784) * [Lens] Functional tests (#44279) Foundational layer for lens functional tests. Lens-specific page objects are not in this PR. * [Lens] Add Lens visualizations to Visualize list (#43398) * [Lens] Suggestion improvements (#43688) * [lens] Calculate existence of fields in datasource * [lens] Show field details on click * [lens] Calculate existence of fields in datasource (#44422) * [lens] Calculate existence of fields in datasource * Fix route registration * Add page object and use existence in functional test * Simplify layout of filters for index pattern * Respond to review feedback * Update class names * Use new URL constant * Fix usage of base path * Fix lint errors * Add basic API test and make trigger into draggable * [Lens ] Preview metric (#43755) * format filter ratio as percentage (#44625) * [Lens] Remove datasource suggestion id (#44495) * [Lens] Make breadcrumbs look and feel like Visualize (#44258) * Improve click target for popover * [lens] Fix breakage from app-arch movements (#44720) * [lens] Fix type error in test from merge * Merge branch origin/feature/lens * [lens] Fix registration of embeddable (#45171) * Upgrade EUI and fix typescript breakages * Undo EUI changes * [Lens] Functional tests (#44814) Basic functional tests for Lens, by no means comprehensive. This is more of a smokescreen test of some normal use cases. * Fix snapshots * Fix issues with types and tests * Add back accidentally committed code * [lens] Add Lens to CODEOWNERS (#45296) * Change popover styles and simplify types on server * [lens] Fix visualization alias registration * [lens] Fix usage of EUI after typescript upgrade (#45404) * [lens] Fix usage of EUI after typescript upgrade * Use local fix instead of workaround * Process results more * Fix API errors * Fix API tests and use sampleCount * Simplify document count calculation * [lens] Fix usage of expressions plugin (#45544) * [lens] Fix usage of expressions plugin * Use updated exports from #45538 * Fix imports and mocha tests * Use relative instead of absolute path to fix tests * Fix API test * Fix type errors and pass through new platform from top * [lens] More cleanup from QueryBar changes in master (#45687) * Fix tests * Fix i18n issue * [lens] Fix build and use new platform from entry points (#45834) * [lens] Fix build and use new platform from entry points * Fix params for existence route * Fix tests and add boolean/empty field support * Updated top values * Fixed up overall layout including adding a popover header * Fix up time display * Improve popover edge cases and respond to feedback * Fix field formatters for histogram * Use custom style for empty top values --- .../plugins/apm/typings/elasticsearch.ts | 15 +- x-pack/legacy/plugins/lens/common/api.ts | 36 ++ x-pack/legacy/plugins/lens/common/index.ts | 1 + .../indexpattern_plugin/_field_itm.scss | 21 + .../public/indexpattern_plugin/datapanel.tsx | 3 + .../public/indexpattern_plugin/field_item.tsx | 474 +++++++++++++++++- .../indexpattern_plugin/indexpattern.scss | 7 +- .../plugins/lens/server/routes/field_stats.ts | 277 ++++++++++ .../plugins/lens/server/routes/index.ts | 2 + .../plugins/lens/server/routes/index_stats.ts | 9 +- .../api_integration/apis/lens/field_stats.ts | 266 ++++++++++ .../test/api_integration/apis/lens/index.ts | 1 + .../api_integration/apis/lens/index_stats.ts | 14 + 13 files changed, 1104 insertions(+), 22 deletions(-) create mode 100644 x-pack/legacy/plugins/lens/common/api.ts create mode 100644 x-pack/legacy/plugins/lens/public/indexpattern_plugin/_field_itm.scss create mode 100644 x-pack/legacy/plugins/lens/server/routes/field_stats.ts create mode 100644 x-pack/test/api_integration/apis/lens/field_stats.ts diff --git a/x-pack/legacy/plugins/apm/typings/elasticsearch.ts b/x-pack/legacy/plugins/apm/typings/elasticsearch.ts index 86ea7d3bcb4bd..7d63d1ede2022 100644 --- a/x-pack/legacy/plugins/apm/typings/elasticsearch.ts +++ b/x-pack/legacy/plugins/apm/typings/elasticsearch.ts @@ -28,7 +28,9 @@ declare module 'elasticsearch' { | 'extended_stats' | 'filter' | 'filters' - | 'cardinality'; + | 'cardinality' + | 'sampler' + | 'value_count'; type AggOptions = AggregationOptionMap & { [key: string]: any; @@ -71,6 +73,12 @@ declare module 'elasticsearch' { >; }; + type SamplerAggregation = SubAggregation< + SubAggregationMap + > & { + doc_count: number; + }; + interface AggregatedValue { value: number | null; } @@ -82,7 +90,9 @@ declare module 'elasticsearch' { max: AggregatedValue; min: AggregatedValue; sum: AggregatedValue; - terms: BucketAggregation; + value_count: AggregatedValue; + // Elasticsearch might return terms with numbers, but this is a more limited type + terms: BucketAggregation; date_histogram: BucketAggregation< AggregationOption[AggregationName], number @@ -128,6 +138,7 @@ declare module 'elasticsearch' { cardinality: { value: number; }; + sampler: SamplerAggregation; }[AggregationType & keyof AggregationOption[AggregationName]]; } >; diff --git a/x-pack/legacy/plugins/lens/common/api.ts b/x-pack/legacy/plugins/lens/common/api.ts new file mode 100644 index 0000000000000..f7e1c439074fc --- /dev/null +++ b/x-pack/legacy/plugins/lens/common/api.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface BucketedAggregation { + buckets: Array<{ + key: KeyType; + count: number; + }>; +} + +export interface NumberStatsResult { + count: number; + histogram: BucketedAggregation; + topValues: BucketedAggregation; +} + +export interface TopValuesResult { + count: number; + topValues: BucketedAggregation; +} + +export interface FieldStatsResponse { + // Total count of documents + totalDocuments?: number; + // If sampled, the exact number of matching documents + sampledDocuments?: number; + // If sampled, the exact number of values sampled. Can be higher than documents + // because Elasticsearch supports arrays for all fields + sampledValues?: number; + // Histogram and values are based on distinct values, not based on documents + histogram?: BucketedAggregation; + topValues?: BucketedAggregation; +} diff --git a/x-pack/legacy/plugins/lens/common/index.ts b/x-pack/legacy/plugins/lens/common/index.ts index 358d0d5b7e076..eead93dd33480 100644 --- a/x-pack/legacy/plugins/lens/common/index.ts +++ b/x-pack/legacy/plugins/lens/common/index.ts @@ -4,4 +4,5 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './api'; export * from './constants'; diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/_field_itm.scss b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/_field_itm.scss new file mode 100644 index 0000000000000..be957dbb403bf --- /dev/null +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/_field_itm.scss @@ -0,0 +1,21 @@ +.lnsFieldItem__topValue { + margin-bottom: $euiSizeS; + + &:last-of-type { + margin-bottom: 0; + } +} + +.lnsFieldItem__topValueProgress { + background-color: $euiColorLightestShade; + + &::-webkit-progress-bar { + background-color: $euiColorLightestShade; + } +} + +.lnsFieldItem__fieldPopoverPanel { + min-width: 260px; + max-width: 300px; +} + diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx index 9b7a3435d9e50..8009c4ebf3e6d 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx @@ -532,11 +532,14 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ const overallField = fieldByName[field.name]; return ( ); })} diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx index 0afc769688218..4d3d1d378c328 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx @@ -4,17 +4,62 @@ * you may not use this file except in compliance with the Elastic License. */ -import React from 'react'; +import React, { useState } from 'react'; +import DateMath from '@elastic/datemath'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiProgress, + EuiPopover, + EuiLoadingSpinner, + EuiKeyboardAccessible, + EuiText, + EuiToolTip, + EuiButtonGroup, + EuiPopoverFooter, + EuiPopoverTitle, +} from '@elastic/eui'; +import { + Chart, + Axis, + getAxisId, + getSpecId, + BarSeries, + Position, + ScaleType, + Settings, + DataSeriesColorsValues, + TooltipType, + niceTimeFormatter, +} from '@elastic/charts'; +import { i18n } from '@kbn/i18n'; +import { toElasticsearchQuery } from '@kbn/es-query'; +import { Query } from 'src/plugins/data/common'; +// @ts-ignore +import { fieldFormats } from '../../../../../../src/legacy/ui/public/registry/field_formats'; import { IndexPattern, IndexPatternField, DraggedField } from './indexpattern'; import { DragDrop } from '../drag_drop'; -import { FieldIcon } from './field_icon'; -import { DataType } from '..'; +import { FieldIcon, getColorForDataType } from './field_icon'; +import { DatasourceDataPanelProps, DataType } from '../types'; +import { BucketedAggregation, FieldStatsResponse } from '../../common'; export interface FieldItemProps { + core: DatasourceDataPanelProps['core']; field: IndexPatternField; indexPattern: IndexPattern; highlight?: string; exists: boolean; + query: Query; + dateRange: DatasourceDataPanelProps['dateRange']; +} + +interface State { + isLoading: boolean; + totalDocuments?: number; + sampledDocuments?: number; + sampledValues?: number; + histogram?: BucketedAggregation; + topValues?: BucketedAggregation; } function wrapOnDot(str?: string) { @@ -24,7 +69,15 @@ function wrapOnDot(str?: string) { return str ? str.replace(/\./g, '.\u200B') : ''; } -export function FieldItem({ field, indexPattern, highlight, exists }: FieldItemProps) { +export function FieldItem(props: FieldItemProps) { + const { core, field, indexPattern, highlight, exists, query, dateRange } = props; + + const [infoIsOpen, setOpen] = useState(false); + + const [state, setState] = useState({ + isLoading: false, + }); + const wrappableName = wrapOnDot(field.name)!; const wrappableHighlight = wrapOnDot(highlight); const highlightIndex = wrappableHighlight @@ -41,22 +94,407 @@ export function FieldItem({ field, indexPattern, highlight, exists }: FieldItemP ); + function fetchData() { + if ( + state.isLoading || + (field.type !== 'number' && + field.type !== 'string' && + field.type !== 'date' && + field.type !== 'boolean') + ) { + return; + } + + setState(s => ({ ...s, isLoading: true })); + + core.http + .post(`/api/lens/index_stats/${indexPattern.title}/field`, { + body: JSON.stringify({ + query: toElasticsearchQuery(query, indexPattern), + fromDate: dateRange.fromDate, + toDate: dateRange.toDate, + timeFieldName: indexPattern.timeFieldName, + field, + }), + }) + .then((results: FieldStatsResponse) => { + setState(s => ({ + ...s, + isLoading: false, + totalDocuments: results.totalDocuments, + sampledDocuments: results.sampledDocuments, + sampledValues: results.sampledValues, + histogram: results.histogram, + topValues: results.topValues, + })); + }) + .catch(() => { + setState(s => ({ ...s, isLoading: false })); + }); + } + + function togglePopover() { + setOpen(!infoIsOpen); + if (!infoIsOpen) { + fetchData(); + } + } + return ( - ('.application') || undefined} + button={ + + +
{ + togglePopover(); + }} + onKeyPress={event => { + if (event.key === 'ENTER') { + togglePopover(); + } + }} + title={i18n.translate('xpack.lens.indexPattern.fieldStatsButton', { + defaultMessage: 'Click or Enter for more information about {fieldName}', + values: { fieldName: field.name }, + })} + aria-label={i18n.translate('xpack.lens.indexPattern.fieldStatsButton', { + defaultMessage: 'Click or Enter for more information about {fieldName}', + values: { fieldName: field.name }, + })} + > + + + + {wrappableHighlightableFieldName} + +
+ + + } + isOpen={infoIsOpen} + closePopover={() => setOpen(false)} + anchorPosition="rightUp" + panelClassName="lnsFieldItem__fieldPopoverPanel" > -
- + + + ); +} + +function FieldItemPopoverContents(props: State & FieldItemProps) { + const { histogram, topValues, indexPattern, field, dateRange, core, sampledValues } = props; + + if (props.isLoading) { + return ; + } else if ( + (!props.histogram || props.histogram.buckets.length === 0) && + (!props.topValues || props.topValues.buckets.length === 0) + ) { + return ( + + {i18n.translate('xpack.lens.indexPattern.fieldStatsNoData', { + defaultMessage: 'No data to display', + })} + + ); + } + + let histogramDefault = !!props.histogram; + + const totalValuesCount = + topValues && topValues.buckets.reduce((prev, bucket) => bucket.count + prev, 0); + const otherCount = sampledValues && totalValuesCount ? sampledValues - totalValuesCount : 0; + + if ( + totalValuesCount && + histogram && + histogram.buckets.length && + topValues && + topValues.buckets.length + ) { + // Default to histogram when top values are less than 10% of total + histogramDefault = otherCount / totalValuesCount > 0.9; + } + + const [showingHistogram, setShowingHistogram] = useState(histogramDefault); + + let formatter: { convert: (data: unknown) => string }; + if (indexPattern.fieldFormatMap && indexPattern.fieldFormatMap[field.name]) { + const FormatType = fieldFormats.getType(indexPattern.fieldFormatMap[field.name].id); + if (FormatType) { + formatter = new FormatType( + indexPattern.fieldFormatMap[field.name].params, + core.uiSettings.get.bind(core.uiSettings) + ); + } else { + formatter = { convert: (data: unknown) => JSON.stringify(data) }; + } + } else { + formatter = fieldFormats.getDefaultInstance(field.type, field.esTypes); + } + + const euiButtonColor = + field.type === 'string' ? 'accent' : field.type === 'number' ? 'secondary' : 'primary'; + const euiTextColor = + field.type === 'string' ? 'accent' : field.type === 'number' ? 'secondary' : 'default'; + + const fromDate = DateMath.parse(dateRange.fromDate); + const toDate = DateMath.parse(dateRange.toDate); - - {wrappableHighlightableFieldName} - + let title = <>; + + if (histogram && histogram.buckets.length && topValues && topValues.buckets.length) { + title = ( + { + setShowingHistogram(optionId === 'histogram'); + }} + idSelected={showingHistogram ? 'histogram' : 'topValues'} + /> + ); + } else if (field.type === 'date') { + title = ( + <> + {i18n.translate('xpack.lens.indexPattern.fieldTimeDistributionLabel', { + defaultMessage: 'Time distribution', + })} + + ); + } else if (topValues && topValues.buckets.length) { + title = ( + <> + {i18n.translate('xpack.lens.indexPattern.fieldTopValuesLabel', { + defaultMessage: 'Top Values', + })} + + ); + } + + function wrapInPopover(el: React.ReactElement) { + return ( + <> + {title ? {title} : <>} + {el} + + {props.totalDocuments ? ( + + + {props.sampledDocuments && ( + <> + {i18n.translate('xpack.lens.indexPattern.percentageOfLabel', { + defaultMessage: '{percentage}% of', + values: { + percentage: Math.round((props.sampledDocuments / props.totalDocuments) * 100), + }, + })} + + )}{' '} + + {fieldFormats + .getDefaultInstance('number', ['integer']) + .convert(props.totalDocuments)} + {' '} + {i18n.translate('xpack.lens.indexPattern.ofDocumentsLabel', { + defaultMessage: 'documents', + })} + + + ) : ( + <> + )} + + ); + } + + if (histogram && histogram.buckets.length) { + const specId = getSpecId( + i18n.translate('xpack.lens.indexPattern.fieldStatsCountLabel', { + defaultMessage: 'Count', + }) + ); + const expectedColor = getColorForDataType(field.type); + const colors: DataSeriesColorsValues = { + colorValues: [], + specId, + }; + + const seriesColors = new Map([[colors, expectedColor]]); + + if (field.type === 'date') { + return wrapInPopover( + + + + + + + + ); + } else if (showingHistogram || !topValues || !topValues.buckets.length) { + return wrapInPopover( + + + + formatter.convert(d)} + /> + + + + ); + } + } + + if (props.topValues && props.topValues.buckets.length) { + return wrapInPopover( +
+ {props.topValues.buckets.map(topValue => { + const formatted = formatter.convert(topValue.key); + return ( +
+ + + {formatted === '' ? ( + + + {i18n.translate('xpack.lens.indexPattern.fieldPanelEmptyStringValue', { + defaultMessage: 'Empty string', + })} + + + ) : ( + + + {formatted} + + + )} + + + + {Math.round((topValue.count / props.sampledValues!) * 100)}% + + + + + +
+ ); + })} + {otherCount ? ( + <> + + + + {i18n.translate('xpack.lens.indexPattern.otherDocsLabel', { + defaultMessage: 'Other', + })} + + + + + + {Math.round((otherCount / props.sampledValues!) * 100)}% + + + + + + + ) : ( + <> + )}
- - ); + ); + } + return <>; } diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss index dcc579dd05ec6..733a30858dab7 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss @@ -1,3 +1,4 @@ +@import './field_itm'; @import './dimension_panel/index'; @@ -52,7 +53,6 @@ @include euiFontSizeS; background: $euiColorEmptyShade; border-radius: $euiBorderRadius; - padding: $euiSizeS; margin-bottom: $euiSizeXS; transition: box-shadow $euiAnimSpeedFast $euiAnimSlightResistance; @@ -70,3 +70,8 @@ .lnsFieldListPanel__fieldName { margin-left: $euiSizeXS; } + +.lnsFieldListPanel__fieldInfo { + padding: $euiSizeS; + font-weight: $euiFontWeightMedium; +} diff --git a/x-pack/legacy/plugins/lens/server/routes/field_stats.ts b/x-pack/legacy/plugins/lens/server/routes/field_stats.ts new file mode 100644 index 0000000000000..a57811362c6cf --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/routes/field_stats.ts @@ -0,0 +1,277 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; +import DateMath from '@elastic/datemath'; +import { schema } from '@kbn/config-schema'; +import { AggregationSearchResponse } from 'elasticsearch'; +import { CoreSetup } from 'src/core/server'; +import { FieldStatsResponse } from '../../common'; + +const SHARD_SIZE = 5000; + +export async function initFieldsRoute(setup: CoreSetup) { + const router = setup.http.createRouter(); + router.post( + { + path: '/index_stats/{indexPatternTitle}/field', + validate: { + params: schema.object({ + indexPatternTitle: schema.string(), + }), + body: schema.object( + { + query: schema.object({}, { allowUnknowns: true }), + fromDate: schema.string(), + toDate: schema.string(), + timeFieldName: schema.string(), + field: schema.object( + { + name: schema.string(), + type: schema.string(), + esTypes: schema.maybe(schema.arrayOf(schema.string())), + }, + { allowUnknowns: true } + ), + }, + { allowUnknowns: true } + ), + }, + }, + async (context, req, res) => { + const requestClient = context.core.elasticsearch.dataClient; + const { fromDate, toDate, timeFieldName, field, query } = req.body; + + try { + const filters = { + bool: { + filter: [ + { + range: { + [timeFieldName]: { + gte: fromDate, + lte: toDate, + }, + }, + }, + query, + ], + }, + }; + + const search = (aggs: unknown) => + requestClient.callAsCurrentUser('search', { + index: req.params.indexPatternTitle, + body: { + query: filters, + aggs, + }, + // The hits total changed in 7.0 from number to object, unless this flag is set + // this is a workaround for elasticsearch response types that are from 6.x + restTotalHitsAsInt: true, + size: 0, + }); + + if (field.type === 'number') { + return res.ok({ + body: await getNumberHistogram(search, field), + }); + } else if (field.type === 'string') { + return res.ok({ + body: await getStringSamples(search, field), + }); + } else if (field.type === 'date') { + return res.ok({ + body: await getDateHistogram(search, field, { fromDate, toDate }), + }); + } else if (field.type === 'boolean') { + return res.ok({ + body: await getStringSamples(search, field), + }); + } + + return res.ok({}); + } catch (e) { + if (e.status === 404) { + return res.notFound(); + } + if (e.isBoom) { + if (e.output.statusCode === 404) { + return res.notFound(); + } + return res.internalError(e.output.message); + } else { + return res.internalError({ + body: Boom.internal(e.message || e.name), + }); + } + } + } + ); +} + +export async function getNumberHistogram( + aggSearchWithBody: (body: unknown) => Promise, + field: { name: string; type: string; esTypes?: string[] } +): Promise { + const searchBody = { + sample: { + sampler: { shard_size: SHARD_SIZE }, + aggs: { + min_value: { + min: { field: field.name }, + }, + max_value: { + max: { field: field.name }, + }, + sample_count: { value_count: { field: field.name } }, + top_values: { + terms: { field: field.name, size: 10 }, + }, + }, + }, + }; + + const minMaxResult = (await aggSearchWithBody(searchBody)) as AggregationSearchResponse< + unknown, + { body: { aggs: typeof searchBody } } + >; + + const minValue = minMaxResult.aggregations!.sample.min_value.value; + const maxValue = minMaxResult.aggregations!.sample.max_value.value; + const terms = minMaxResult.aggregations!.sample.top_values; + const topValuesBuckets = { + buckets: terms.buckets.map(bucket => ({ + count: bucket.doc_count, + key: bucket.key, + })), + }; + + let histogramInterval = (maxValue! - minValue!) / 10; + + if (Number.isInteger(minValue!) && Number.isInteger(maxValue!)) { + histogramInterval = Math.ceil(histogramInterval); + } + + if (histogramInterval === 0) { + return { + totalDocuments: minMaxResult.hits.total, + sampledValues: minMaxResult.aggregations!.sample.sample_count.value!, + sampledDocuments: minMaxResult.aggregations!.sample.doc_count, + topValues: topValuesBuckets, + histogram: { buckets: [] }, + }; + } + + const histogramBody = { + sample: { + sampler: { shard_size: SHARD_SIZE }, + aggs: { + histo: { + histogram: { + field: field.name, + interval: histogramInterval, + }, + }, + }, + }, + }; + const histogramResult = (await aggSearchWithBody(histogramBody)) as AggregationSearchResponse< + unknown, + { body: { aggs: typeof histogramBody } } + >; + + return { + totalDocuments: minMaxResult.hits.total, + sampledDocuments: minMaxResult.aggregations!.sample.doc_count, + sampledValues: minMaxResult.aggregations!.sample.sample_count.value!, + histogram: { + buckets: histogramResult.aggregations!.sample.histo.buckets.map(bucket => ({ + count: bucket.doc_count, + key: bucket.key, + })), + }, + topValues: topValuesBuckets, + }; +} + +export async function getStringSamples( + aggSearchWithBody: (body: unknown) => unknown, + field: { name: string; type: string } +): Promise { + const topValuesBody = { + sample: { + sampler: { shard_size: SHARD_SIZE }, + aggs: { + sample_count: { value_count: { field: field.name } }, + top_values: { + terms: { field: field.name, size: 10 }, + }, + }, + }, + }; + const topValuesResult = (await aggSearchWithBody(topValuesBody)) as AggregationSearchResponse< + unknown, + { body: { aggs: typeof topValuesBody } } + >; + + return { + totalDocuments: topValuesResult.hits.total, + sampledDocuments: topValuesResult.aggregations!.sample.doc_count, + sampledValues: topValuesResult.aggregations!.sample.sample_count.value!, + topValues: { + buckets: topValuesResult.aggregations!.sample.top_values.buckets.map(bucket => ({ + count: bucket.doc_count, + key: bucket.key, + })), + }, + }; +} + +// This one is not sampled so that it returns the full date range +export async function getDateHistogram( + aggSearchWithBody: (body: unknown) => unknown, + field: { name: string; type: string }, + range: { fromDate: string; toDate: string } +): Promise { + const fromDate = DateMath.parse(range.fromDate); + const toDate = DateMath.parse(range.toDate); + if (!fromDate) { + throw Error('Invalid fromDate value'); + } + if (!toDate) { + throw Error('Invalid toDate value'); + } + + const interval = Math.round((toDate.valueOf() - fromDate.valueOf()) / 10); + if (interval < 1) { + return { + totalDocuments: 0, + histogram: { buckets: [] }, + }; + } + + // TODO: Respect rollup intervals + const fixedInterval = `${interval}ms`; + + const histogramBody = { + histo: { date_histogram: { field: field.name, fixed_interval: fixedInterval } }, + }; + const results = (await aggSearchWithBody(histogramBody)) as AggregationSearchResponse< + unknown, + { body: { aggs: typeof histogramBody } } + >; + + return { + totalDocuments: results.hits.total, + histogram: { + buckets: results.aggregations!.histo.buckets.map(bucket => ({ + count: bucket.doc_count, + key: bucket.key, + })), + }, + }; +} diff --git a/x-pack/legacy/plugins/lens/server/routes/index.ts b/x-pack/legacy/plugins/lens/server/routes/index.ts index 9a957765cc87d..c5f882c8e5714 100644 --- a/x-pack/legacy/plugins/lens/server/routes/index.ts +++ b/x-pack/legacy/plugins/lens/server/routes/index.ts @@ -6,7 +6,9 @@ import { CoreSetup } from 'src/core/server'; import { initStatsRoute } from './index_stats'; +import { initFieldsRoute } from './field_stats'; export function setupRoutes(setup: CoreSetup) { initStatsRoute(setup); + initFieldsRoute(setup); } diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts index aeb213e356786..d746de0a2878a 100644 --- a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts @@ -73,6 +73,7 @@ export async function initStatsRoute(setup: CoreSetup) { }, ], }, + // TODO: Add script_fields here once saved objects are available on the server }, size, }, @@ -85,8 +86,14 @@ export async function initStatsRoute(setup: CoreSetup) { } return res.ok({ body: {} }); } catch (e) { + if (e.status === 404) { + return res.notFound(); + } if (e.isBoom) { - return res.internalError(e); + if (e.output.statusCode === 404) { + return res.notFound(); + } + return res.internalError(e.output.message); } else { return res.internalError({ body: Boom.internal(e.message || e.name), diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts new file mode 100644 index 0000000000000..9eba9392c4f7f --- /dev/null +++ b/x-pack/test/api_integration/apis/lens/field_stats.ts @@ -0,0 +1,266 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +const TEST_START_TIME = '2015-09-19T06:31:44.000'; +const TEST_END_TIME = '2015-09-23T18:31:44.000'; +const COMMON_HEADERS = { + 'kbn-xsrf': 'some-xsrf-token', +}; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + + describe('index stats apis', () => { + before(async () => { + await esArchiver.loadIfNeeded('logstash_functional'); + await esArchiver.loadIfNeeded('visualize/default'); + }); + after(async () => { + await esArchiver.unload('logstash_functional'); + await esArchiver.unload('visualize/default'); + }); + + describe('field distribution', () => { + it('should return a 404 for missing index patterns', async () => { + await supertest + .post('/api/lens/index_stats/logstash/field') + .set(COMMON_HEADERS) + .send({ + query: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: 'bytes', + type: 'number', + }, + }) + .expect(404); + }); + + it('should return an auto histogram for numbers and top values', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + query: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: 'bytes', + type: 'number', + }, + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4633, + sampledDocuments: 4633, + sampledValues: 4633, + histogram: { + buckets: [ + { + count: 705, + key: 0, + }, + { + count: 898, + key: 1999, + }, + { + count: 885, + key: 3998, + }, + { + count: 970, + key: 5997, + }, + { + count: 939, + key: 7996, + }, + { + count: 44, + key: 9995, + }, + { + count: 43, + key: 11994, + }, + { + count: 43, + key: 13993, + }, + { + count: 57, + key: 15992, + }, + { + count: 49, + key: 17991, + }, + ], + }, + topValues: { + buckets: [ + { + count: 147, + key: 0, + }, + { + count: 5, + key: 3954, + }, + { + count: 5, + key: 6497, + }, + { + count: 4, + key: 1840, + }, + { + count: 4, + key: 4206, + }, + { + count: 4, + key: 4328, + }, + { + count: 4, + key: 4669, + }, + { + count: 4, + key: 5846, + }, + { + count: 4, + key: 5863, + }, + { + count: 4, + key: 6631, + }, + ], + }, + }); + }); + + it('should return an auto histogram for dates', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + query: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: '@timestamp', + type: 'date', + }, + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4633, + histogram: { + buckets: [ + { + count: 1161, + key: 1442875680000, + }, + { + count: 3420, + key: 1442914560000, + }, + { + count: 52, + key: 1442953440000, + }, + ], + }, + }); + }); + + it('should return top values for strings', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + query: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: 'geo.src', + type: 'string', + }, + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4633, + sampledDocuments: 4633, + sampledValues: 4633, + topValues: { + buckets: [ + { + count: 832, + key: 'CN', + }, + { + count: 804, + key: 'IN', + }, + { + count: 425, + key: 'US', + }, + { + count: 158, + key: 'ID', + }, + { + count: 143, + key: 'BR', + }, + { + count: 116, + key: 'PK', + }, + { + count: 106, + key: 'BD', + }, + { + count: 94, + key: 'NG', + }, + { + count: 84, + key: 'RU', + }, + { + count: 73, + key: 'JP', + }, + ], + }, + }); + }); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/lens/index.ts b/x-pack/test/api_integration/apis/lens/index.ts index 9827eadb1278b..d8c02db99b10a 100644 --- a/x-pack/test/api_integration/apis/lens/index.ts +++ b/x-pack/test/api_integration/apis/lens/index.ts @@ -9,5 +9,6 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function lensApiIntegrationTests({ loadTestFile }: FtrProviderContext) { describe('Lens', () => { loadTestFile(require.resolve('./index_stats')); + loadTestFile(require.resolve('./field_stats')); }); } diff --git a/x-pack/test/api_integration/apis/lens/index_stats.ts b/x-pack/test/api_integration/apis/lens/index_stats.ts index 8dc181fa9b601..17ab6a813e480 100644 --- a/x-pack/test/api_integration/apis/lens/index_stats.ts +++ b/x-pack/test/api_integration/apis/lens/index_stats.ts @@ -105,6 +105,20 @@ export default ({ getService }: FtrProviderContext) => { expect(Object.keys(body)).to.eql(fieldsWithData.map(field => field.name)); }); + + it('should throw a 404 for a non-existent index', async () => { + await supertest + .post('/api/lens/index_stats/fake') + .set(COMMON_HEADERS) + .send({ + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + size: 500, + fields: [], + }) + .expect(404); + }); }); }); };