From a102bab430db9d43754755b9d5f71901c097abc7 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 22 Feb 2021 11:37:52 -0700 Subject: [PATCH 01/26] rename ml:fileDataVisualizerMaxFileSize to fileUppload:maxFileSize --- .../plugins/file_upload/common/constants.ts | 3 ++ .../plugins/file_upload/public/api/index.ts | 2 ++ .../file_upload/public/get_max_bytes.ts | 31 +++++++++++++++++++ .../file_upload/public/kibana_services.ts | 1 + x-pack/plugins/file_upload/public/plugin.ts | 3 ++ x-pack/plugins/file_upload/server/plugin.ts | 23 ++++++++++++++ .../plugins/ml/common/constants/settings.ts | 1 - .../datavisualizer_selector.tsx | 5 ++- .../about_panel/welcome_content.tsx | 4 +-- .../file_datavisualizer_view.js | 4 +-- .../file_based/components/utils/index.ts | 2 -- .../file_based/components/utils/utils.ts | 26 +--------------- .../ml/server/lib/register_settings.ts | 20 ------------ 13 files changed, 70 insertions(+), 55 deletions(-) create mode 100644 x-pack/plugins/file_upload/public/get_max_bytes.ts diff --git a/x-pack/plugins/file_upload/common/constants.ts b/x-pack/plugins/file_upload/common/constants.ts index 11ad80f5c955..f64626b5a70a 100644 --- a/x-pack/plugins/file_upload/common/constants.ts +++ b/x-pack/plugins/file_upload/common/constants.ts @@ -5,6 +5,9 @@ * 2.0. */ +export const UI_SETTING_MAX_FILE_SIZE = 'fileUpload:maxFileSize'; +// export const FILE_DATA_VISUALIZER_MAX_FILE_SIZE = 'ml:fileDataVisualizerMaxFileSize'; + export const MB = Math.pow(2, 20); export const MAX_FILE_SIZE = '100MB'; export const MAX_FILE_SIZE_BYTES = 104857600; // 100MB diff --git a/x-pack/plugins/file_upload/public/api/index.ts b/x-pack/plugins/file_upload/public/api/index.ts index 359bc4a1687b..8884c398fa2e 100644 --- a/x-pack/plugins/file_upload/public/api/index.ts +++ b/x-pack/plugins/file_upload/public/api/index.ts @@ -12,6 +12,8 @@ import type { IImporter, ImportFactoryOptions } from '../importer'; export interface FileUploadStartApi { getFileUploadComponent(): Promise>; importerFactory(format: string, options: ImportFactoryOptions): Promise; + getMaxBytes(): number; + getMaxBytesFormatted(): string; } export async function getFileUploadComponent(): Promise< diff --git a/x-pack/plugins/file_upload/public/get_max_bytes.ts b/x-pack/plugins/file_upload/public/get_max_bytes.ts new file mode 100644 index 000000000000..2e002e65248c --- /dev/null +++ b/x-pack/plugins/file_upload/public/get_max_bytes.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// @ts-ignore +import numeral from '@elastic/numeral'; +import { + MAX_FILE_SIZE, + MAX_FILE_SIZE_BYTES, + ABSOLUTE_MAX_FILE_SIZE_BYTES, + FILE_SIZE_DISPLAY_FORMAT, + UI_SETTING_MAX_FILE_SIZE, +} from '../common'; +import { getUiSettings } from './kibana_services'; + +export function getMaxBytes() { + const maxFileSize = getUiSettings().get(UI_SETTING_MAX_FILE_SIZE, MAX_FILE_SIZE); + // @ts-ignore + const maxBytes = numeral(maxFileSize.toUpperCase()).value(); + if (maxBytes < MAX_FILE_SIZE_BYTES) { + return MAX_FILE_SIZE_BYTES; + } + return maxBytes <= ABSOLUTE_MAX_FILE_SIZE_BYTES ? maxBytes : ABSOLUTE_MAX_FILE_SIZE_BYTES; +} + +export function getMaxBytesFormatted() { + return numeral(getMaxBytes()).format(FILE_SIZE_DISPLAY_FORMAT); +} diff --git a/x-pack/plugins/file_upload/public/kibana_services.ts b/x-pack/plugins/file_upload/public/kibana_services.ts index c007c5c2273a..a604136ca34e 100644 --- a/x-pack/plugins/file_upload/public/kibana_services.ts +++ b/x-pack/plugins/file_upload/public/kibana_services.ts @@ -18,3 +18,4 @@ export function setStartServices(core: CoreStart, plugins: FileUploadStartDepend export const getIndexPatternService = () => pluginsStart.data.indexPatterns; export const getHttp = () => coreStart.http; export const getSavedObjectsClient = () => coreStart.savedObjects.client; +export const getUiSettings = () => coreStart.uiSettings; diff --git a/x-pack/plugins/file_upload/public/plugin.ts b/x-pack/plugins/file_upload/public/plugin.ts index 5d3918193d48..af3dd65b788a 100644 --- a/x-pack/plugins/file_upload/public/plugin.ts +++ b/x-pack/plugins/file_upload/public/plugin.ts @@ -9,6 +9,7 @@ import { CoreStart, Plugin } from '../../../../src/core/public'; import { FileUploadStartApi, getFileUploadComponent, importerFactory } from './api'; import { setStartServices } from './kibana_services'; import { DataPublicPluginStart } from '../../../../src/plugins/data/public'; +import { getMaxBytes, getMaxBytesFormatted } from './get_max_bytes'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface FileUploadSetupDependencies {} @@ -34,6 +35,8 @@ export class FileUploadPlugin return { getFileUploadComponent, importerFactory, + getMaxBytes, + getMaxBytesFormatted, }; } } diff --git a/x-pack/plugins/file_upload/server/plugin.ts b/x-pack/plugins/file_upload/server/plugin.ts index 74a18633d8a2..80021b0ec43e 100644 --- a/x-pack/plugins/file_upload/server/plugin.ts +++ b/x-pack/plugins/file_upload/server/plugin.ts @@ -5,10 +5,13 @@ * 2.0. */ +import { i18n } from '@kbn/i18n'; import { CoreSetup, CoreStart, Plugin } from 'src/core/server'; +import { schema } from '@kbn/config-schema'; import { fileUploadRoutes } from './routes'; import { initFileUploadTelemetry } from './telemetry'; import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; +import { UI_SETTING_MAX_FILE_SIZE, MAX_FILE_SIZE } from '../common'; interface SetupDeps { usageCollection: UsageCollectionSetup; @@ -18,6 +21,26 @@ export class FileUploadPlugin implements Plugin { async setup(coreSetup: CoreSetup, plugins: SetupDeps) { fileUploadRoutes(coreSetup.http.createRouter()); + coreSetup.uiSettings.register({ + [UI_SETTING_MAX_FILE_SIZE]: { + name: i18n.translate('xpack.fileUpload.maxFileSizeUiSetting.name', { + defaultMessage: 'Maximum file upload size', + }), + value: MAX_FILE_SIZE, + description: i18n.translate('xpack.fileUpload.maxFileSizeUiSetting.description', { + defaultMessage: + 'Sets the file size limit when importing files. The highest supported value for this setting is 1GB.', + }), + schema: schema.string(), + validation: { + regexString: '\\d+[mMgG][bB]', + message: i18n.translate('xpack.ml.maxFileSizeUiSetting.error', { + defaultMessage: 'Should be a valid data size. e.g. 200MB, 1GB', + }), + }, + }, + }); + initFileUploadTelemetry(coreSetup, plugins.usageCollection); } diff --git a/x-pack/plugins/ml/common/constants/settings.ts b/x-pack/plugins/ml/common/constants/settings.ts index 30e0e0a57afe..5a9b18a232dc 100644 --- a/x-pack/plugins/ml/common/constants/settings.ts +++ b/x-pack/plugins/ml/common/constants/settings.ts @@ -5,7 +5,6 @@ * 2.0. */ -export const FILE_DATA_VISUALIZER_MAX_FILE_SIZE = 'ml:fileDataVisualizerMaxFileSize'; export const ANOMALY_DETECTION_ENABLE_TIME_RANGE = 'ml:anomalyDetection:results:enableTimeDefaults'; export const ANOMALY_DETECTION_DEFAULT_TIME_RANGE = 'ml:anomalyDetection:results:timeDefaults'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx index 79d17a7846b8..3f620ae16d57 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx @@ -25,10 +25,9 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { isFullLicense } from '../license'; import { useTimefilter, useMlKibana, useNavigateToPath } from '../contexts/kibana'; - import { NavigationMenu } from '../components/navigation_menu'; -import { getMaxBytesFormatted } from './file_based/components/utils'; import { HelpMenu } from '../components/help_menu'; +import { getFileUpload } from '../util/dependency_cache'; function startTrialDescription() { return ( @@ -68,7 +67,7 @@ export const DatavisualizerSelector: FC = () => { licenseManagement.enabled === true && isFullLicense() === false; - const maxFileSize = getMaxBytesFormatted(); + const maxFileSize = getFileUpload().getMaxBytesFormatted(); return ( diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx index f20319e60668..6748f2e21e3d 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx @@ -20,7 +20,7 @@ import { } from '@elastic/eui'; import { ExperimentalBadge } from '../experimental_badge'; -import { getMaxBytesFormatted } from '../utils'; +import { getFileUpload } from '../../../../util/dependency_cache'; export const WelcomeContent: FC = () => { const toolTipContent = i18n.translate( @@ -30,7 +30,7 @@ export const WelcomeContent: FC = () => { } ); - const maxFileSize = getMaxBytesFormatted(); + const maxFileSize = getFileUpload().getMaxBytesFormatted(); return ( diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js index 8b683a154519..90c2b2bf1863 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js @@ -22,12 +22,12 @@ import { ExplanationFlyout } from '../explanation_flyout'; import { ImportView } from '../import_view'; import { DEFAULT_LINES_TO_SAMPLE, - getMaxBytes, readFile, createUrlOverrides, processResults, hasImportPermission, } from '../utils'; +import { getFileUpload } from '../../../../util/dependency_cache'; import { MODE } from './constants'; @@ -60,7 +60,7 @@ export class FileDataVisualizerView extends Component { this.originalSettings = { linesToSample: DEFAULT_LINES_TO_SAMPLE, }; - this.maxFileUploadBytes = getMaxBytes(); + this.maxFileUploadBytes = getFileUpload().getMaxBytes(); } async componentDidMount() { diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.ts index 41ea5582a7de..209f2ef5ad17 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.ts @@ -10,7 +10,5 @@ export { hasImportPermission, processResults, readFile, - getMaxBytes, - getMaxBytesFormatted, DEFAULT_LINES_TO_SAMPLE, } from './utils'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts index 2c1b02b53354..ebde771603fc 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts @@ -6,19 +6,9 @@ */ import { isEqual } from 'lodash'; -// @ts-ignore -import numeral from '@elastic/numeral'; import { ml } from '../../../../services/ml_api_service'; import { AnalysisResult, InputOverrides } from '../../../../../../common/types/file_datavisualizer'; -import { - MB, - MAX_FILE_SIZE, - MAX_FILE_SIZE_BYTES, - ABSOLUTE_MAX_FILE_SIZE_BYTES, - FILE_SIZE_DISPLAY_FORMAT, -} from '../../../../../../../file_upload/public'; -import { getUiSettings } from '../../../../util/dependency_cache'; -import { FILE_DATA_VISUALIZER_MAX_FILE_SIZE } from '../../../../../../common/constants/settings'; +import { MB } from '../../../../../../../file_upload/public'; export const DEFAULT_LINES_TO_SAMPLE = 1000; const UPLOAD_SIZE_MB = 5; @@ -66,20 +56,6 @@ export function readFile(file: File) { }); } -export function getMaxBytes() { - const maxFileSize = getUiSettings().get(FILE_DATA_VISUALIZER_MAX_FILE_SIZE, MAX_FILE_SIZE); - // @ts-ignore - const maxBytes = numeral(maxFileSize.toUpperCase()).value(); - if (maxBytes < MAX_FILE_SIZE_BYTES) { - return MAX_FILE_SIZE_BYTES; - } - return maxBytes <= ABSOLUTE_MAX_FILE_SIZE_BYTES ? maxBytes : ABSOLUTE_MAX_FILE_SIZE_BYTES; -} - -export function getMaxBytesFormatted() { - return numeral(getMaxBytes()).format(FILE_SIZE_DISPLAY_FORMAT); -} - export function createUrlOverrides(overrides: InputOverrides, originalSettings: InputOverrides) { const formattedOverrides: InputOverrides = {}; for (const o in overrideDefaults) { diff --git a/x-pack/plugins/ml/server/lib/register_settings.ts b/x-pack/plugins/ml/server/lib/register_settings.ts index 5f9107bd2981..8f279114344e 100644 --- a/x-pack/plugins/ml/server/lib/register_settings.ts +++ b/x-pack/plugins/ml/server/lib/register_settings.ts @@ -9,34 +9,14 @@ import { CoreSetup } from 'kibana/server'; import { i18n } from '@kbn/i18n'; import { schema } from '@kbn/config-schema'; import { - FILE_DATA_VISUALIZER_MAX_FILE_SIZE, ANOMALY_DETECTION_DEFAULT_TIME_RANGE, ANOMALY_DETECTION_ENABLE_TIME_RANGE, DEFAULT_AD_RESULTS_TIME_FILTER, DEFAULT_ENABLE_AD_RESULTS_TIME_FILTER, } from '../../common/constants/settings'; -import { MAX_FILE_SIZE } from '../../../file_upload/common'; export function registerKibanaSettings(coreSetup: CoreSetup) { coreSetup.uiSettings.register({ - [FILE_DATA_VISUALIZER_MAX_FILE_SIZE]: { - name: i18n.translate('xpack.ml.maxFileSizeSettingsName', { - defaultMessage: 'File Data Visualizer maximum file upload size', - }), - value: MAX_FILE_SIZE, - description: i18n.translate('xpack.ml.maxFileSizeSettingsDescription', { - defaultMessage: - 'Sets the file size limit when importing data in the File Data Visualizer. The highest supported value for this setting is 1GB.', - }), - category: ['machineLearning'], - schema: schema.string(), - validation: { - regexString: '\\d+[mMgG][bB]', - message: i18n.translate('xpack.ml.maxFileSizeSettingsError', { - defaultMessage: 'Should be a valid data size. e.g. 200MB, 1GB', - }), - }, - }, [ANOMALY_DETECTION_ENABLE_TIME_RANGE]: { name: i18n.translate('xpack.ml.advancedSettings.enableAnomalyDetectionDefaultTimeRangeName', { defaultMessage: 'Enable time filter defaults for anomaly detection results', From bf4bf6a2c0fe843fb12270597b9afd780c7f144e Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 22 Feb 2021 11:46:21 -0700 Subject: [PATCH 02/26] add saved object migration --- .../saved_objects/migrations.test.ts | 34 +++++++++++++++++++ .../ui_settings/saved_objects/migrations.ts | 19 +++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/core/server/ui_settings/saved_objects/migrations.test.ts b/src/core/server/ui_settings/saved_objects/migrations.test.ts index ea5b4133b49c..7b53f4e21dd9 100644 --- a/src/core/server/ui_settings/saved_objects/migrations.test.ts +++ b/src/core/server/ui_settings/saved_objects/migrations.test.ts @@ -44,3 +44,37 @@ describe('ui_settings 7.9.0 migrations', () => { }); }); }); + +describe('ui_settings 7.13.0 migrations', () => { + const migration = migrations['7.13.0']; + + test('returns doc on empty object', () => { + expect(migration({} as SavedObjectUnsanitizedDoc)).toEqual({ + references: [], + }); + }); + test('properly renames ml:fileDataVisualizerMaxFileSize to fileUpload:maxFileSize', () => { + const doc = { + type: 'config', + id: '8.0.0', + attributes: { + buildNum: 9007199254740991, + 'ml:fileDataVisualizerMaxFileSize': '250MB', + }, + references: [], + updated_at: '2020-06-09T20:18:20.349Z', + migrationVersion: {}, + }; + expect(migration(doc)).toEqual({ + type: 'config', + id: '8.0.0', + attributes: { + buildNum: 9007199254740991, + 'fileUpload:maxFileSize': '250MB', + }, + references: [], + updated_at: '2020-06-09T20:18:20.349Z', + migrationVersion: {}, + }); + }); +}); diff --git a/src/core/server/ui_settings/saved_objects/migrations.ts b/src/core/server/ui_settings/saved_objects/migrations.ts index 7ea5573076ba..b5dc13fcdae3 100644 --- a/src/core/server/ui_settings/saved_objects/migrations.ts +++ b/src/core/server/ui_settings/saved_objects/migrations.ts @@ -28,4 +28,23 @@ export const migrations = { }), references: doc.references || [], }), + '7.13.0': (doc: SavedObjectUnsanitizedDoc): SavedObjectSanitizedDoc => ({ + ...doc, + ...(doc.attributes && { + attributes: Object.keys(doc.attributes).reduce( + (acc, key) => + key === 'ml:fileDataVisualizerMaxFileSize' + ? { + ...acc, + ['fileUpload:maxFileSize']: doc.attributes[key], + } + : { + ...acc, + [key]: doc.attributes[key], + }, + {} + ), + }), + references: doc.references || [], + }), }; From 1db54f70a735e83023defc76fe61ba311bc42930 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 23 Feb 2021 11:21:05 -0700 Subject: [PATCH 03/26] file preview --- .../public/components/geojson_file_picker.tsx | 152 ++++++++++++ .../components/json_index_file_picker.js | 10 +- .../components/json_upload_and_parse.js | 50 ++-- .../geojson_importer/geojson_importer.ts | 216 +++++++++--------- .../public/importer/geojson_importer/index.ts | 2 +- .../public/importer/validate_file.ts | 52 +++++ 6 files changed, 349 insertions(+), 133 deletions(-) create mode 100644 x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx create mode 100644 x-pack/plugins/file_upload/public/importer/validate_file.ts diff --git a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx new file mode 100644 index 000000000000..041efd5cf407 --- /dev/null +++ b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { Component } from 'react'; +import { EuiFilePicker, EuiFormRow } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { MB } from '../../common'; +import { getMaxBytes, getMaxBytesFormatted } from '../get_max_bytes'; +import { GeoJsonImporter, GEOJSON_FILE_TYPES } from '../importer/geojson_importer'; + +interface Props { + onSelect: ({ + features, + geoFieldTypes, + importer, + indexName, + }: { + features: Feature[]; + indexName: string; + importer: Importer; + geoFieldTypes: string[]; + }) => void; + onClear: () => void; +} + +interface State { + error: string | null; + isLoadingPreview: boolean; + previewSummary: string | null; +} + +export class GeoJsonFilePicker extends Component { + private _isMounted = false; + + state: State = { + error: null, + isLoadingPreview: false, + previewSummary: null, + }; + + async componentDidMount() { + this._isMounted = true; + } + + componentWillUnmount() { + this._isMounted = false; + } + + _onFileSelect = async (files: FileList) => { + this.props.onClear(); + + this.setState({ + error: null, + isLoadingPreview: false, + previewSummary: null, + }); + + if (files.length === 0) { + return; + } + + const file = files[0]; + this.setState({ isLoadingPreview: true }); + + let importer: GeoJsonImporter; + let previewError: string | null = null; + let preview: { features: Feature[]; geoFieldTypes: string[]; previewCoverage: number }; + try { + importer = new GeoJsonImporter(file); + preview = await importer.previewFile(10000, MB * 3); + if (preview.features.length === 0) { + previewError = i18n.translate('xpack.fileUpload.geojsonFilePicker.noFeaturesDetected', { + defaultMessage: 'No GeoJson features found in selected file.', + }); + } + } catch (error) { + previewError = error.message; + } + + if (!this._isMounted) { + return; + } + + this.setState({ + error: previewError, + isLoadingPreview: false, + previewSummary: + previewError === null + ? i18n.translate('xpack.fileUpload.geojsonFilePicker.previewSummary', { + defaultMessage: 'Previewing {numFeatures} features, {previewCoverage}% of file.', + values: { + numFeatures: preview.features.length, + previewCoverage: preview.previewCoverage.toFixed(2), + }, + }) + : null, + }); + + if (importer && preview) { + this.props.onSelect({ + ...preview, + importer, + indexName: file.name.split('.')[0], + }); + } + }; + + _renderHelpText() { + return this.state.previewSummary !== null ? ( + this.state.previewSummary + ) : ( + + {i18n.translate('xpack.fileUpload.geojsonFilePicker.acceptedFormats', { + defaultMessage: 'Formats accepted: {fileTypes}', + values: { fileTypes: GEOJSON_FILE_TYPES.join(', ') }, + })} +
+ {i18n.translate('xpack.fileUpload.geojsonFilePicker.maxSize', { + defaultMessage: 'Max size: {maxFileSize}', + values: { maxFileSize: getMaxBytesFormatted() }, + })} +
+ {i18n.translate('xpack.fileUpload.geojsonFilePicker.acceptedCoordinateSystem', { + defaultMessage: 'Coordinates must be in EPSG:4326 coordinate reference system.', + })} + + ); + } + + render() { + return ( + + + + ); + } +} diff --git a/x-pack/plugins/file_upload/public/components/json_index_file_picker.js b/x-pack/plugins/file_upload/public/components/json_index_file_picker.js index 78bf7378578d..fcbc3b9432c6 100644 --- a/x-pack/plugins/file_upload/public/components/json_index_file_picker.js +++ b/x-pack/plugins/file_upload/public/components/json_index_file_picker.js @@ -9,8 +9,8 @@ import React, { Fragment, Component } from 'react'; import { EuiFilePicker, EuiFormRow, EuiProgress } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; +import { getMaxBytes, getMaxBytesFormatted } from '../get_max_bytes'; -const MAX_FILE_SIZE = 52428800; const ACCEPTABLE_FILETYPES = ['json', 'geojson']; const acceptedFileTypeString = ACCEPTABLE_FILETYPES.map((type) => `.${type}`).join(','); const acceptedFileTypeStringMessage = ACCEPTABLE_FILETYPES.map((type) => `.${type}`).join(', '); @@ -35,6 +35,7 @@ export class JsonIndexFilePicker extends Component { isFileParseActive = () => this._isMounted && this.state.fileParseActive; _fileHandler = (fileList) => { + console.log('fileList', fileList); const fileArr = Array.from(fileList); this.props.resetFileAndIndexSettings(); this.setState({ @@ -114,13 +115,13 @@ export class JsonIndexFilePicker extends Component { const { currentFileTracker } = this.state; const { setFileRef, setParsedFile, resetFileAndIndexSettings } = this.props; - if (file.size > MAX_FILE_SIZE) { + if (file.size > getMaxBytes()) { this.setState({ fileUploadError: i18n.translate('xpack.fileUpload.jsonIndexFilePicker.acceptableFileSize', { defaultMessage: 'File size {fileSize} exceeds maximum file size of {maxFileSize}', values: { fileSize: bytesToSize(file.size), - maxFileSize: bytesToSize(MAX_FILE_SIZE), + maxFileSize: getMaxBytesFormatted(), }, }), }); @@ -128,6 +129,7 @@ export class JsonIndexFilePicker extends Component { return; } + console.log('file', file); const defaultIndexName = this._getFileNameAndCheckType(file); if (!defaultIndexName) { resetFileAndIndexSettings(); @@ -231,7 +233,7 @@ export class JsonIndexFilePicker extends Component { id="xpack.fileUpload.jsonIndexFilePicker.maxSize" defaultMessage="Max size: {maxFileSize}" values={{ - maxFileSize: bytesToSize(MAX_FILE_SIZE), + maxFileSize: getMaxBytesFormatted(), }} />
diff --git a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js index d4f6858eb599..94c359836836 100644 --- a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js +++ b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js @@ -10,12 +10,11 @@ import { i18n } from '@kbn/i18n'; import { EuiForm } from '@elastic/eui'; import PropTypes from 'prop-types'; import { IndexSettings } from './index_settings'; -import { JsonIndexFilePicker } from './json_index_file_picker'; import { JsonImportProgress } from './json_import_progress'; import _ from 'lodash'; -import { GeoJsonImporter } from '../importer/geojson_importer'; import { ES_FIELD_TYPES } from '../../../../../src/plugins/data/public'; import { getIndexPatternService } from '../kibana_services'; +import { GeoJsonFilePicker } from './geojson_file_picker'; const INDEXING_STAGE = { INDEXING_STARTED: i18n.translate('xpack.fileUpload.jsonUploadAndParse.dataIndexingStarted', { @@ -44,11 +43,8 @@ const INDEXING_STAGE = { }; export class JsonUploadAndParse extends Component { - geojsonImporter = new GeoJsonImporter(); - state = { // File state - fileRef: null, parsedFile: null, indexedFile: null, @@ -74,19 +70,24 @@ export class JsonUploadAndParse extends Component { componentWillUnmount() { this._isMounted = false; + if (this._geojsonImporter) { + this._geojsonImporter.destroy(); + this._geojsonImporter = null; + } } _resetFileAndIndexSettings = () => { - if (this.props.onFileRemove && this.state.fileRef) { - this.props.onFileRemove(this.state.fileRef); + if (this._geojsonImporter) { + this._geojsonImporter.destroy(); + this._geojsonImporter = undefined; } + + this.props.onFileRemove(); + this.setState({ indexTypes: [], selectedIndexType: '', indexName: '', - indexedFile: null, - parsedFile: null, - fileRef: null, }); }; @@ -271,12 +272,25 @@ export class JsonUploadAndParse extends Component { }); }; + _onFileSelect = ({ features, geoFieldTypes, importer, indexName }) => { + this._geojsonImporter = importer; + + this.setState({ indexTypes: geoFieldTypes }); + + this.props.onFileUpload( + { + type: 'FeatureCollection', + features, + }, + indexName + ); + }; + render() { const { currentIndexingStage, indexDataResp, indexPatternResp, - fileRef, indexName, indexTypes, showImportProgress, @@ -297,18 +311,12 @@ export class JsonUploadAndParse extends Component { /> ) : ( - this.setState({ fileRef })} - setParsedFile={(parsedFile, indexName) => { - this.setState({ parsedFile, indexName }); - this.props.onFileUpload(parsedFile.parsedGeojson, indexName); - }} - resetFileAndIndexSettings={this._resetFileAndIndexSettings} - geojsonImporter={this.geojsonImporter} + this.setState({ indexName })} indexTypes={indexTypes} diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts index 189084e9180d..19c6d2273e6b 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts @@ -5,6 +5,7 @@ * 2.0. */ +import _ from 'lodash'; import { Feature, FeatureCollection, @@ -23,10 +24,117 @@ import { Importer } from '../importer'; import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; // @ts-expect-error import { geoJsonCleanAndValidate } from './geojson_clean_and_validate'; +import { validateFile } from '../validate_file'; + +export const GEOJSON_FILE_TYPES = ['.json', '.geojson']; export class GeoJsonImporter extends Importer { - constructor() { + private _file: File; + private _isActive = true; + private _iterator?: Iterator; + private _hasNext = true; + private _features: Feature[] = []; + private _totalBytesProcessed = 0; + private _unimportedBytesProcessed = 0; + private _totalFeatures = 0; + private _geometryTypesMap = new Map(); + private _invalidCount = 0; + private _prevBatchLastFeature?: Feature; + + constructor(file: File) { super(); + + validateFile(file, GEOJSON_FILE_TYPES); + this._file = file; + } + + public destroy() { + this._isActive = false; + } + + public async previewFile( + rowLimit?: number, + sizeLimit?: number + ): { features: Feature[]; geoFieldTypes: string[]; previewCoverage: number } { + await this._readUntil(rowLimit, sizeLimit); + return { + features: [...this._features], + previewCoverage: (this._unimportedBytesProcessed / this._file.size) * 100, + geoFieldTypes: + this._geometryTypesMap.has('Point') || this._geometryTypesMap.has('MultiPoint') + ? [ES_FIELD_TYPES.GEO_POINT, ES_FIELD_TYPES.GEO_SHAPE] + : [ES_FIELD_TYPES.GEO_SHAPE], + }; + } + + private async _readUntil(rowLimit?: number, sizeLimit?: number) { + while ( + this._isActive && + this._hasNext && + (rowLimit === undefined || this._features.length < rowLimit) && + (sizeLimit === undefined || this._unimportedBytesProcessed < sizeLimit) + ) { + await this._next(); + } + } + + private async _next() { + if (this._iterator === undefined) { + this._iterator = await loadInBatches(this._file, JSONLoader, { + json: { + jsonpaths: ['$.features'], + _rootObjectBatches: true, + }, + }); + } + + if (!this._isActive) { + return; + } + + const { value: batch, done } = await this._iterator.next(); + + if (!this._isActive || done) { + this._hasNext = false; + return; + } + + if ('bytesUsed' in batch) { + const bytesRead = batch.bytesUsed - this._totalBytesProcessed; + this._unimportedBytesProcessed += bytesRead; + this._totalBytesProcessed = batch.bytesUsed; + } + + const rawFeatures: unknown[] = this._prevBatchLastFeature ? [this._prevBatchLastFeature] : []; + this._prevBatchLastFeature = undefined; + const isLastBatch = batch.batchType === 'root-object-batch-complete'; + if (isLastBatch) { + // Handle single feature geoJson + if (this._totalFeatures === 0) { + rawFeatures.push(batch.container); + } + } else { + rawFeatures.push(...batch.data); + } + + for (let i = 0; i < rawFeatures.length; i++) { + const rawFeature = rawFeatures[i] as Feature; + if (!isLastBatch && i === rawFeatures.length - 1) { + // Do not process last feature until next batch is read, features on batch boundary may be incomplete. + this._prevBatchLastFeature = rawFeature; + continue; + } + + this._totalFeatures++; + if (!rawFeature.geometry || !rawFeature.geometry.type) { + this._invalidCount++; + } else { + if (!this._geometryTypesMap.has(rawFeature.geometry.type)) { + this._geometryTypesMap.set(rawFeature.geometry.type, true); + } + this._features.push(geoJsonCleanAndValidate(rawFeature)); + } + } } public read(data: ArrayBuffer): { success: boolean } { @@ -69,110 +177,4 @@ export class GeoJsonImporter extends Importer { }); } } - - public async readFile( - file: File, - setFileProgress: ({ - featuresProcessed, - bytesProcessed, - totalBytes, - }: { - featuresProcessed: number; - bytesProcessed: number; - totalBytes: number; - }) => void, - isFileParseActive: () => boolean - ): Promise<{ - errors: string[]; - geometryTypes: string[]; - parsedGeojson: FeatureCollection; - } | null> { - if (!file) { - throw new Error( - i18n.translate('xpack.fileUpload.fileParser.noFileProvided', { - defaultMessage: 'Error, no file provided', - }) - ); - } - - return new Promise(async (resolve, reject) => { - const batches = await loadInBatches(file, JSONLoader, { - json: { - jsonpaths: ['$.features'], - _rootObjectBatches: true, - }, - }); - - const rawFeatures: unknown[] = []; - for await (const batch of batches) { - if (!isFileParseActive()) { - break; - } - - if (batch.batchType === 'root-object-batch-complete') { - // Handle single feature geoJson - if (rawFeatures.length === 0) { - rawFeatures.push(batch.container); - } - } else { - rawFeatures.push(...batch.data); - } - - setFileProgress({ - featuresProcessed: rawFeatures.length, - bytesProcessed: batch.bytesUsed, - totalBytes: file.size, - }); - } - - if (!isFileParseActive()) { - resolve(null); - return; - } - - if (rawFeatures.length === 0) { - reject( - new Error( - i18n.translate('xpack.fileUpload.fileParser.noFeaturesDetected', { - defaultMessage: 'Error, no features detected', - }) - ) - ); - return; - } - - const features: Feature[] = []; - const geometryTypesMap = new Map(); - let invalidCount = 0; - for (let i = 0; i < rawFeatures.length; i++) { - const rawFeature = rawFeatures[i] as Feature; - if (!rawFeature.geometry || !rawFeature.geometry.type) { - invalidCount++; - } else { - if (!geometryTypesMap.has(rawFeature.geometry.type)) { - geometryTypesMap.set(rawFeature.geometry.type, true); - } - features.push(geoJsonCleanAndValidate(rawFeature)); - } - } - - const errors: string[] = []; - if (invalidCount > 0) { - errors.push( - i18n.translate('xpack.fileUpload.fileParser.featuresOmitted', { - defaultMessage: '{invalidCount} features without geometry omitted', - values: { invalidCount }, - }) - ); - } - resolve({ - errors, - geometryTypes: Array.from(geometryTypesMap.keys()), - parsedGeojson: { - type: 'FeatureCollection', - features, - }, - }); - }); - } } diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/index.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/index.ts index 9ffb84e60316..fccf1a5d9ae2 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/index.ts +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export { GeoJsonImporter } from './geojson_importer'; +export { GeoJsonImporter, GEOJSON_FILE_TYPES } from './geojson_importer'; diff --git a/x-pack/plugins/file_upload/public/importer/validate_file.ts b/x-pack/plugins/file_upload/public/importer/validate_file.ts new file mode 100644 index 000000000000..c83d55d71db1 --- /dev/null +++ b/x-pack/plugins/file_upload/public/importer/validate_file.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { getMaxBytes, getMaxBytesFormatted } from '../get_max_bytes'; + +export function validateFile(file: File, types: string[]) { + if (file.size > getMaxBytes()) { + throw new Error( + i18n.translate('xpack.fileUpload.fileSizeError', { + defaultMessage: 'File size {fileSize} exceeds maximum file size of {maxFileSize}', + values: { + fileSize: bytesToSize(file.size), + maxFileSize: getMaxBytesFormatted(), + }, + }) + ); + } + + if (!file.name) { + throw new Error( + i18n.translate('xpack.fileUpload.noFileNameError', { + defaultMessage: 'File name not provided', + }) + ); + } + + const nameSplit = file.name.split('.'); + const fileType = nameSplit.pop(); + if (!types.includes(`.${fileType}`)) { + throw new Error( + i18n.translate('xpack.fileUpload.fileTypeError', { + defaultMessage: 'File is not one of acceptable types: {types}', + values: { + types: types.join(', '), + }, + }) + ); + } +} + +function bytesToSize(bytes) { + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; + if (bytes === 0) return 'n/a'; + const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)), 10); + if (i === 0) return `${bytes} ${sizes[i]})`; + return `${(bytes / 1024 ** i).toFixed(1)} ${sizes[i]}`; +} From eb63959a4aa46891502b5a673269638af702a9f8 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 23 Feb 2021 14:57:22 -0700 Subject: [PATCH 04/26] importing status --- .../public/components/geojson_file_picker.tsx | 2 +- .../components/json_index_file_picker.js | 269 ---------------- .../components/json_upload_and_parse.js | 293 ++++++++---------- .../geojson_importer/geojson_importer.ts | 167 ++++++++-- .../file_upload/public/importer/importer.ts | 4 +- 5 files changed, 259 insertions(+), 476 deletions(-) delete mode 100644 x-pack/plugins/file_upload/public/components/json_index_file_picker.js diff --git a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx index 041efd5cf407..21bc2a1b3446 100644 --- a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx +++ b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx @@ -94,7 +94,7 @@ export class GeoJsonFilePicker extends Component { defaultMessage: 'Previewing {numFeatures} features, {previewCoverage}% of file.', values: { numFeatures: preview.features.length, - previewCoverage: preview.previewCoverage.toFixed(2), + previewCoverage: preview.previewCoverage, }, }) : null, diff --git a/x-pack/plugins/file_upload/public/components/json_index_file_picker.js b/x-pack/plugins/file_upload/public/components/json_index_file_picker.js deleted file mode 100644 index fcbc3b9432c6..000000000000 --- a/x-pack/plugins/file_upload/public/components/json_index_file_picker.js +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { Fragment, Component } from 'react'; -import { EuiFilePicker, EuiFormRow, EuiProgress } from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { i18n } from '@kbn/i18n'; -import { getMaxBytes, getMaxBytesFormatted } from '../get_max_bytes'; - -const ACCEPTABLE_FILETYPES = ['json', 'geojson']; -const acceptedFileTypeString = ACCEPTABLE_FILETYPES.map((type) => `.${type}`).join(','); -const acceptedFileTypeStringMessage = ACCEPTABLE_FILETYPES.map((type) => `.${type}`).join(', '); - -export class JsonIndexFilePicker extends Component { - state = { - fileUploadError: '', - percentageProcessed: 0, - featuresProcessed: 0, - fileParseActive: false, - currentFileTracker: null, - }; - - async componentDidMount() { - this._isMounted = true; - } - - componentWillUnmount() { - this._isMounted = false; - } - - isFileParseActive = () => this._isMounted && this.state.fileParseActive; - - _fileHandler = (fileList) => { - console.log('fileList', fileList); - const fileArr = Array.from(fileList); - this.props.resetFileAndIndexSettings(); - this.setState({ - fileUploadError: '', - percentageProcessed: 0, - featuresProcessed: 0, - }); - if (fileArr.length === 0) { - // Remove - this.setState({ - fileParseActive: false, - }); - return; - } - const file = fileArr[0]; - - this.setState( - { - fileParseActive: true, - currentFileTracker: Symbol(), - }, - () => this._parseFile(file) - ); - }; - - _getFileNameAndCheckType({ name }) { - let fileNameOnly; - try { - if (!name) { - throw new Error( - i18n.translate('xpack.fileUpload.jsonIndexFilePicker.noFileNameError', { - defaultMessage: 'No file name provided', - }) - ); - } - - const splitNameArr = name.split('.'); - const fileType = splitNameArr.pop(); - if (!ACCEPTABLE_FILETYPES.includes(fileType)) { - //should only occur if browser does not accept the accept parameter - throw new Error( - i18n.translate('xpack.fileUpload.jsonIndexFilePicker.acceptableTypesError', { - defaultMessage: 'File is not one of acceptable types: {types}', - values: { - types: ACCEPTABLE_FILETYPES.join(', '), - }, - }) - ); - } - - fileNameOnly = splitNameArr[0]; - } catch (error) { - this.setState({ - fileUploadError: i18n.translate( - 'xpack.fileUpload.jsonIndexFilePicker.fileProcessingError', - { - defaultMessage: 'File processing error: {errorMessage}', - values: { - errorMessage: error.message, - }, - } - ), - }); - return; - } - return fileNameOnly.toLowerCase(); - } - - setFileProgress = ({ featuresProcessed, bytesProcessed, totalBytes }) => { - const percentageProcessed = parseInt((100 * bytesProcessed) / totalBytes); - if (this.isFileParseActive()) { - this.setState({ featuresProcessed, percentageProcessed }); - } - }; - - async _parseFile(file) { - const { currentFileTracker } = this.state; - const { setFileRef, setParsedFile, resetFileAndIndexSettings } = this.props; - - if (file.size > getMaxBytes()) { - this.setState({ - fileUploadError: i18n.translate('xpack.fileUpload.jsonIndexFilePicker.acceptableFileSize', { - defaultMessage: 'File size {fileSize} exceeds maximum file size of {maxFileSize}', - values: { - fileSize: bytesToSize(file.size), - maxFileSize: getMaxBytesFormatted(), - }, - }), - }); - resetFileAndIndexSettings(); - return; - } - - console.log('file', file); - const defaultIndexName = this._getFileNameAndCheckType(file); - if (!defaultIndexName) { - resetFileAndIndexSettings(); - return; - } - - const fileResult = await this.props.geojsonImporter - .readFile(file, this.setFileProgress, this.isFileParseActive) - .catch((err) => { - if (this._isMounted) { - this.setState({ - fileParseActive: false, - percentageProcessed: 0, - featuresProcessed: 0, - fileUploadError: ( - - ), - }); - resetFileAndIndexSettings(); - return; - } - }); - - if (!this._isMounted) { - return; - } - - // If another file is replacing this one, leave file parse active - this.setState({ - percentageProcessed: 0, - featuresProcessed: 0, - fileParseActive: currentFileTracker !== this.state.currentFileTracker, - }); - if (!fileResult) { - resetFileAndIndexSettings(); - return; - } - - if (fileResult.errors.length) { - this.setState({ - fileUploadError: ( - - ), - }); - } - setFileRef(file); - setParsedFile(fileResult, defaultIndexName); - } - - render() { - const { fileUploadError, percentageProcessed, featuresProcessed } = this.state; - - return ( - - {percentageProcessed ? ( - - ) : null} - - } - isInvalid={fileUploadError !== ''} - error={[fileUploadError]} - helpText={ - percentageProcessed ? ( - i18n.translate('xpack.fileUpload.jsonIndexFilePicker.parsingFile', { - defaultMessage: '{featuresProcessed} features parsed...', - values: { - featuresProcessed, - }, - }) - ) : ( - - {i18n.translate('xpack.fileUpload.jsonIndexFilePicker.formatsAccepted', { - defaultMessage: 'Formats accepted: {acceptedFileTypeStringMessage}', - values: { - acceptedFileTypeStringMessage, - }, - })}{' '} -
- -
- {i18n.translate('xpack.fileUpload.jsonIndexFilePicker.coordinateSystemAccepted', { - defaultMessage: 'Coordinates must be in EPSG:4326 coordinate reference system.', - })}{' '} - - ) - } - > - - } - onChange={this._fileHandler} - accept={acceptedFileTypeString} - /> - - - ); - } -} - -function bytesToSize(bytes) { - const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; - if (bytes === 0) return 'n/a'; - const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)), 10); - if (i === 0) return `${bytes} ${sizes[i]})`; - return `${(bytes / 1024 ** i).toFixed(1)} ${sizes[i]}`; -} diff --git a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js index 94c359836836..4937c438df86 100644 --- a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js +++ b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js @@ -7,59 +7,37 @@ import React, { Component, Fragment } from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiForm } from '@elastic/eui'; +import { EuiForm, EuiProgress, EuiText } from '@elastic/eui'; import PropTypes from 'prop-types'; import { IndexSettings } from './index_settings'; -import { JsonImportProgress } from './json_import_progress'; -import _ from 'lodash'; -import { ES_FIELD_TYPES } from '../../../../../src/plugins/data/public'; import { getIndexPatternService } from '../kibana_services'; import { GeoJsonFilePicker } from './geojson_file_picker'; -const INDEXING_STAGE = { - INDEXING_STARTED: i18n.translate('xpack.fileUpload.jsonUploadAndParse.dataIndexingStarted', { - defaultMessage: 'Data indexing started', - }), - WRITING_TO_INDEX: i18n.translate('xpack.fileUpload.jsonUploadAndParse.writingToIndex', { - defaultMessage: 'Writing to index', - }), - INDEXING_COMPLETE: i18n.translate('xpack.fileUpload.jsonUploadAndParse.indexingComplete', { - defaultMessage: 'Indexing complete', - }), - CREATING_INDEX_PATTERN: i18n.translate( - 'xpack.fileUpload.jsonUploadAndParse.creatingIndexPattern', - { defaultMessage: 'Creating index pattern' } - ), - INDEX_PATTERN_COMPLETE: i18n.translate( - 'xpack.fileUpload.jsonUploadAndParse.indexPatternComplete', - { defaultMessage: 'Index pattern complete' } - ), - INDEXING_ERROR: i18n.translate('xpack.fileUpload.jsonUploadAndParse.dataIndexingError', { - defaultMessage: 'Data indexing error', - }), - INDEX_PATTERN_ERROR: i18n.translate('xpack.fileUpload.jsonUploadAndParse.indexPatternError', { - defaultMessage: 'Index pattern error', - }), +const PHASE = { + CONFIGURE: 'CONFIGURE', + IMPORT: 'IMPORT', + COMPLETE: 'COMPLETE', }; +function getWritingToIndexMsg(progress) { + return i18n.translate('xpack.fileUpload.jsonUploadAndParse.writingToIndex', { + defaultMessage: 'Writing to index: {progress}% complete', + values: { progress }, + }); +} + export class JsonUploadAndParse extends Component { state = { - // File state - parsedFile: null, - indexedFile: null, - // Index state indexTypes: [], selectedIndexType: '', indexName: '', - indexRequestInFlight: false, - indexPatternRequestInFlight: false, hasIndexErrors: false, isIndexReady: false, // Progress-tracking state - showImportProgress: false, - currentIndexingStage: INDEXING_STAGE.INDEXING_STARTED, + importStatus: '', + phase: PHASE.CONFIGURE, indexDataResp: '', indexPatternResp: '', }; @@ -76,6 +54,13 @@ export class JsonUploadAndParse extends Component { } } + componentDidUpdate() { + this._setIndexReady({ ...this.state, ...this.props }); + if (this.props.isIndexingTriggered && this.state.phase === PHASE.CONFIGURE) { + this._import(); + } + } + _resetFileAndIndexSettings = () => { if (this._geojsonImporter) { this._geojsonImporter.destroy(); @@ -91,54 +76,14 @@ export class JsonUploadAndParse extends Component { }); }; - componentDidUpdate() { - this._updateIndexType(); - this._setIndexReady({ ...this.state, ...this.props }); - this._indexData({ ...this.state, ...this.props }); - if (this.props.isIndexingTriggered && !this.state.showImportProgress && this._isMounted) { - this.setState({ showImportProgress: true }); - } - } - - _updateIndexType() { - let nextIndexTypes = []; - if (this.state.parsedFile) { - nextIndexTypes = - this.state.parsedFile.geometryTypes.includes('Point') || - this.state.parsedFile.geometryTypes.includes('MultiPoint') - ? [ES_FIELD_TYPES.GEO_POINT, ES_FIELD_TYPES.GEO_SHAPE] - : [ES_FIELD_TYPES.GEO_SHAPE]; - } - if (!_.isEqual(nextIndexTypes, this.state.indexTypes)) { - this.setState({ indexTypes: nextIndexTypes }); - } - - if (!this.state.selectedIndexType && nextIndexTypes.length) { - // auto select index type - this.setState({ selectedIndexType: nextIndexTypes[0] }); - } else if ( - this.state.selectedIndexType && - !nextIndexTypes.includes(this.state.selectedIndexType) - ) { - // unselected indexType if selected type is not longer an option - this.setState({ selectedIndexType: null }); - } - } - - _setIndexReady = ({ - parsedFile, - selectedIndexType, - indexName, - hasIndexErrors, - indexRequestInFlight, - onIndexReady, - }) => { + _setIndexReady = ({ selectedIndexType, indexName, hasIndexErrors, phase, onIndexReady }) => { const isIndexReady = - !!parsedFile && + this._geojsonImporter !== undefined && !!selectedIndexType && !!indexName && !hasIndexErrors && - !indexRequestInFlight; + phase === PHASE.CONFIGURE; + console.log('isIndexReady', isIndexReady); if (isIndexReady !== this.state.isIndexReady) { this.setState({ isIndexReady }); if (onIndexReady) { @@ -147,30 +92,10 @@ export class JsonUploadAndParse extends Component { } }; - _indexData = async ({ - indexedFile, - parsedFile, - indexRequestInFlight, - indexName, - selectedIndexType, - isIndexingTriggered, - isIndexReady, - onIndexingComplete, - onIndexingError, - }) => { - // Check index ready - const filesAreEqual = _.isEqual(indexedFile, parsedFile); - if (!isIndexingTriggered || filesAreEqual || !isIndexReady || indexRequestInFlight) { - return; - } - this.setState({ - indexRequestInFlight: true, - currentIndexingStage: INDEXING_STAGE.WRITING_TO_INDEX, - }); - - this.geojsonImporter.setDocs(parsedFile.parsedGeojson, selectedIndexType); - - // initialize import + _import = async () => { + // + // create index + // const settings = { number_of_shards: 1, }; @@ -182,8 +107,16 @@ export class JsonUploadAndParse extends Component { }, }; const ingestPipeline = {}; - const initializeImportResp = await this.geojsonImporter.initializeImport( - indexName, + this.setState({ + importStatus: i18n.translate('xpack.fileUpload.jsonUploadAndParse.dataIndexingStarted', { + defaultMessage: 'Creating index: {indexName}', + values: { indexName: this.state.indexName }, + }), + phase: PHASE.IMPORT, + }); + this._geojsonImporter.setGeoFieldType(this.state.selectedIndexType); + const initializeImportResp = await this._geojsonImporter.initializeImport( + this.state.indexName, settings, mappings, ingestPipeline @@ -193,80 +126,93 @@ export class JsonUploadAndParse extends Component { } if (initializeImportResp.index === undefined || initializeImportResp.id === undefined) { this.setState({ - indexRequestInFlight: false, - currentIndexingStage: INDEXING_STAGE.INDEXING_ERROR, + phase: PHASE.COMPLETE, }); - this._resetFileAndIndexSettings(); - onIndexingError(); + this.props.onIndexingError(); return; } + // // import file - const importResp = await this.geojsonImporter.import( + // + this.setState({ + importStatus: getWritingToIndexMsg(0), + }); + const importResp = await this._geojsonImporter.import( initializeImportResp.id, - indexName, + this.state.indexName, initializeImportResp.pipelineId, - () => {} + (progress) => { + if (this._isMounted) { + this.setState({ + importStatus: getWritingToIndexMsg(progress), + }); + } + } ); if (!this._isMounted) { return; } + if (!importResp.success) { this.setState({ indexDataResp: importResp, - indexRequestInFlight: false, - currentIndexingStage: INDEXING_STAGE.INDEXING_ERROR, + importStatus: i18n.translate('xpack.fileUpload.jsonUploadAndParse.dataIndexingError', { + defaultMessage: 'Data indexing error', + }), + phase: PHASE.COMPLETE, }); - this._resetFileAndIndexSettings(); - onIndexingError(); + this.props.onIndexingError(); return; } - this.setState({ - indexDataResp: importResp, - indexedFile: parsedFile, - currentIndexingStage: INDEXING_STAGE.INDEXING_COMPLETE, - }); + // // create index pattern + // this.setState({ - indexPatternRequestInFlight: true, - currentIndexingStage: INDEXING_STAGE.CREATING_INDEX_PATTERN, + indexDataResp: importResp, + importStatus: i18n.translate('xpack.fileUpload.jsonUploadAndParse.creatingIndexPattern', { + defaultMessage: 'Creating index pattern: {indexName}', + values: { indexName: this.state.indexName }, + }), }); let indexPattern; try { indexPattern = await getIndexPatternService().createAndSave( { - title: indexName, + title: this.state.indexName, }, true ); } catch (error) { if (this._isMounted) { this.setState({ - indexPatternRequestInFlight: false, - currentIndexingStage: INDEXING_STAGE.INDEX_PATTERN_ERROR, + importStatus: i18n.translate('xpack.fileUpload.jsonUploadAndParse.indexPatternError', { + defaultMessage: 'Index pattern error', + }), + phase: PHASE.COMPLETE, }); - this._resetFileAndIndexSettings(); - onIndexingError(); + this.props.onIndexingError(); } return; } if (!this._isMounted) { return; } + + // + // Successful import + // this.setState({ indexPatternResp: { success: true, id: indexPattern.id, fields: indexPattern.fields, }, - indexPatternRequestInFlight: false, + phase: PHASE.COMPLETE, + importStatus: '', }); - this.setState({ - currentIndexingStage: INDEXING_STAGE.INDEX_PATTERN_COMPLETE, - }); - this._resetFileAndIndexSettings(); - onIndexingComplete({ + this.props.onIndexingComplete({ indexDataResp: importResp, indexPattern, }); @@ -275,7 +221,21 @@ export class JsonUploadAndParse extends Component { _onFileSelect = ({ features, geoFieldTypes, importer, indexName }) => { this._geojsonImporter = importer; - this.setState({ indexTypes: geoFieldTypes }); + const newState = { + indexTypes: geoFieldTypes, + indexName, + }; + if (!this.state.selectedIndexType && geoFieldTypes.length) { + // auto select index type + newState.selectedIndexType = geoFieldTypes[0]; + } else if ( + this.state.selectedIndexType && + !geoFieldTypes.includes(this.state.selectedIndexType) + ) { + // unselected indexType if selected type is not longer an option + newState.selectedIndexType = ''; + } + this.setState(newState); this.props.onFileUpload( { @@ -287,44 +247,35 @@ export class JsonUploadAndParse extends Component { }; render() { - const { - currentIndexingStage, - indexDataResp, - indexPatternResp, - indexName, - indexTypes, - showImportProgress, - } = this.state; + if (this.state.phase === PHASE.IMPORT) { + return ( + + + +

{this.state.importStatus}

+ + + ); + } + + if (this.state.phase === PHASE.COMPLETE) { + return
complete
; + } return ( - {showImportProgress ? ( - - ) : ( - - - this.setState({ indexName })} - indexTypes={indexTypes} - setSelectedIndexType={(selectedIndexType) => this.setState({ selectedIndexType })} - setHasIndexErrors={(hasIndexErrors) => this.setState({ hasIndexErrors })} - /> - - )} + + this.setState({ indexName })} + indexTypes={this.state.indexTypes} + setSelectedIndexType={(selectedIndexType) => this.setState({ selectedIndexType })} + setHasIndexErrors={(hasIndexErrors) => this.setState({ hasIndexErrors })} + /> ); } diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts index 19c6d2273e6b..59d898d9572a 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts @@ -20,11 +20,12 @@ import { i18n } from '@kbn/i18n'; // @ts-expect-error import { JSONLoader, loadInBatches } from './loaders'; import { CreateDocsResponse } from '../types'; -import { Importer } from '../importer'; +import { callImportRoute, Importer, IMPORT_RETRIES } from '../importer'; import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; // @ts-expect-error import { geoJsonCleanAndValidate } from './geojson_clean_and_validate'; import { validateFile } from '../validate_file'; +import { MB } from '../../../common'; export const GEOJSON_FILE_TYPES = ['.json', '.geojson']; @@ -40,6 +41,7 @@ export class GeoJsonImporter extends Importer { private _geometryTypesMap = new Map(); private _invalidCount = 0; private _prevBatchLastFeature?: Feature; + private _geoFieldType?: ES_FIELD_TYPES.GEO_POINT | ES_FIELD_TYPES.GEO_SHAPE; constructor(file: File) { super(); @@ -59,7 +61,7 @@ export class GeoJsonImporter extends Importer { await this._readUntil(rowLimit, sizeLimit); return { features: [...this._features], - previewCoverage: (this._unimportedBytesProcessed / this._file.size) * 100, + previewCoverage: Math.round((this._unimportedBytesProcessed / this._file.size) * 100), geoFieldTypes: this._geometryTypesMap.has('Point') || this._geometryTypesMap.has('MultiPoint') ? [ES_FIELD_TYPES.GEO_POINT, ES_FIELD_TYPES.GEO_SHAPE] @@ -67,6 +69,108 @@ export class GeoJsonImporter extends Importer { }; } + public setGeoFieldType(geoFieldType: ES_FIELD_TYPES.GEO_POINT | ES_FIELD_TYPES.GEO_SHAPE) { + this._geoFieldType = geoFieldType; + } + + public async import( + id: string, + index: string, + pipelineId: string, + setImportProgress: (progress: number) => void + ): Promise { + if (!id || !index) { + return { + success: false, + error: i18n.translate('xpack.fileUpload.import.noIdOrIndexSuppliedErrorMessage', { + defaultMessage: 'no ID or index supplied', + }), + }; + } + + const ingestPipeline = { + id: pipelineId, + }; + + let success = true; + const failures: ImportFailure[] = []; + let error; + + while (this._features.length > 0 || (this._hasNext && this._isActive)) { + await this._readUntil(undefined, 10 * MB); + if (!this._isActive) { + return { + success: false, + failures, + docCount: this._totalFeatures, + }; + } + + let retries = IMPORT_RETRIES; + let resp: ImportResponse = { + success: false, + failures: [], + docCount: 0, + id: '', + index: '', + pipelineId: '', + }; + const data = toEsDocs(this._features, this._geoFieldType); + const progress = Math.round((this._totalBytesProcessed / this._file.size) * 100); + this._features = []; + this._unimportedBytesProcessed = 0; + + while (resp.success === false && retries > 0) { + try { + resp = await callImportRoute({ + id, + index, + data, + settings: {}, + mappings: {}, + ingestPipeline, + }); + + if (retries < IMPORT_RETRIES) { + // eslint-disable-next-line no-console + console.log(`Retrying import ${IMPORT_RETRIES - retries}`); + } + + retries--; + } catch (err) { + resp.success = false; + resp.error = err; + retries = 0; + } + } + + if (resp.success) { + setImportProgress(progress); + } else { + success = false; + error = resp.error; + failures.push(...resp.failures); + break; + } + + failures.push(...resp.failures); + } + + const result: ImportResults = { + success, + failures, + docCount: this._docArray.length, + }; + + if (success) { + setImportProgress(100); + } else { + result.error = error; + } + + return result; + } + private async _readUntil(rowLimit?: number, sizeLimit?: number) { while ( this._isActive && @@ -144,37 +248,34 @@ export class GeoJsonImporter extends Importer { protected _createDocs(text: string): CreateDocsResponse { throw new Error('_createDocs not implemented.'); } +} - public getDocs() { - return this._docArray; - } - - public setDocs( - featureCollection: FeatureCollection, - geoFieldType: ES_FIELD_TYPES.GEO_POINT | ES_FIELD_TYPES.GEO_SHAPE - ) { - this._docArray = []; - for (let i = 0; i < featureCollection.features.length; i++) { - const feature = featureCollection.features[i]; - const geometry = feature.geometry as - | Point - | MultiPoint - | LineString - | MultiLineString - | Polygon - | MultiPolygon; - const coordinates = - geoFieldType === ES_FIELD_TYPES.GEO_SHAPE - ? { - type: geometry.type.toLowerCase(), - coordinates: geometry.coordinates, - } - : geometry.coordinates; - const properties = feature.properties ? feature.properties : {}; - this._docArray.push({ - coordinates, - ...properties, - }); - } +function toEsDocs( + features: Feature[], + geoFieldType: ES_FIELD_TYPES.GEO_POINT | ES_FIELD_TYPES.GEO_SHAPE +) { + const esDocs = []; + for (let i = 0; i < features.length; i++) { + const feature = features[i]; + const geometry = feature.geometry as + | Point + | MultiPoint + | LineString + | MultiLineString + | Polygon + | MultiPolygon; + const coordinates = + geoFieldType === ES_FIELD_TYPES.GEO_SHAPE + ? { + type: geometry.type.toLowerCase(), + coordinates: geometry.coordinates, + } + : geometry.coordinates; + const properties = feature.properties ? feature.properties : {}; + esDocs.push({ + coordinates, + ...properties, + }); } + return esDocs; } diff --git a/x-pack/plugins/file_upload/public/importer/importer.ts b/x-pack/plugins/file_upload/public/importer/importer.ts index 8bdb465bd69c..2689f3badc39 100644 --- a/x-pack/plugins/file_upload/public/importer/importer.ts +++ b/x-pack/plugins/file_upload/public/importer/importer.ts @@ -22,7 +22,7 @@ import { CreateDocsResponse, IImporter, ImportResults } from './types'; const CHUNK_SIZE = 5000; const MAX_CHUNK_CHAR_COUNT = 1000000; -const IMPORT_RETRIES = 5; +export const IMPORT_RETRIES = 5; const STRING_CHUNKS_MB = 100; export abstract class Importer implements IImporter { @@ -232,7 +232,7 @@ function createDocumentChunks(docArray: ImportDoc[]) { return chunks; } -function callImportRoute({ +export function callImportRoute({ id, index, data, From 1665c45563bea0ae8b0a78bcfc74323c0e178a19 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 23 Feb 2021 15:14:01 -0700 Subject: [PATCH 05/26] remove console statement --- .../public/components/json_upload_and_parse.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js index 4937c438df86..660e00ca7f34 100644 --- a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js +++ b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js @@ -55,7 +55,7 @@ export class JsonUploadAndParse extends Component { } componentDidUpdate() { - this._setIndexReady({ ...this.state, ...this.props }); + this._setIndexReady(); if (this.props.isIndexingTriggered && this.state.phase === PHASE.CONFIGURE) { this._import(); } @@ -76,19 +76,16 @@ export class JsonUploadAndParse extends Component { }); }; - _setIndexReady = ({ selectedIndexType, indexName, hasIndexErrors, phase, onIndexReady }) => { + _setIndexReady = () => { const isIndexReady = this._geojsonImporter !== undefined && - !!selectedIndexType && - !!indexName && - !hasIndexErrors && - phase === PHASE.CONFIGURE; - console.log('isIndexReady', isIndexReady); + !!this.state.selectedIndexType && + !!this.state.indexName && + !this.state.hasIndexErrors && + this.state.phase === PHASE.CONFIGURE; if (isIndexReady !== this.state.isIndexReady) { this.setState({ isIndexReady }); - if (onIndexReady) { - onIndexReady(isIndexReady); - } + this.props.onIndexReady(isIndexReady); } }; From aca546313dbe4f9b57eb92ef95230c59893841d6 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 23 Feb 2021 19:55:44 -0700 Subject: [PATCH 06/26] import complete view --- x-pack/plugins/file_upload/kibana.json | 3 +- .../components/import_complete_view.tsx | 147 ++++++++++++++++++ .../public/components/json_import_progress.js | 135 ---------------- .../components/json_upload_and_parse.js | 51 +++--- .../geojson_importer/geojson_importer.ts | 2 +- 5 files changed, 177 insertions(+), 161 deletions(-) create mode 100644 x-pack/plugins/file_upload/public/components/import_complete_view.tsx delete mode 100644 x-pack/plugins/file_upload/public/components/json_import_progress.js diff --git a/x-pack/plugins/file_upload/kibana.json b/x-pack/plugins/file_upload/kibana.json index 7676a01d0b0f..6d55e4f46d7e 100644 --- a/x-pack/plugins/file_upload/kibana.json +++ b/x-pack/plugins/file_upload/kibana.json @@ -4,5 +4,6 @@ "kibanaVersion": "kibana", "server": true, "ui": true, - "requiredPlugins": ["data", "usageCollection"] + "requiredPlugins": ["data", "usageCollection"], + "requiredBundles": ["kibanaReact"] } diff --git a/x-pack/plugins/file_upload/public/components/import_complete_view.tsx b/x-pack/plugins/file_upload/public/components/import_complete_view.tsx new file mode 100644 index 000000000000..0a9fd5373751 --- /dev/null +++ b/x-pack/plugins/file_upload/public/components/import_complete_view.tsx @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { Component, Fragment } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiCallOut, EuiSpacer, EuiText, EuiTitle } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { CodeEditor, KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public'; +import { getHttp, getUiSettings } from '../kibana_services'; +import { ImportResponse } from '../../common'; + +const services = { + uiSettings: getUiSettings(), +}; + +interface Props { + importResp?: ImportResponse; + indexPatternResp?: unknown; +} + +export class ImportCompleteView extends Component { + _renderCodeEditor(value: unknown) { + return ( +
+ {}} + options={{ + readOnly: true, + lineNumbers: 'off', + fontSize: 12, + minimap: { + enabled: false, + }, + scrollBeyondLastLine: false, + wordWrap: 'on', + wrappingIndent: 'indent', + automaticLayout: true, + }} + /> +
+ ); + } + + _renderIndexResp() { + if (!this.props.importResp) { + return null; + } + + return ( + + +

+ +

+ + {this._renderCodeEditor(this.props.importResp)} + + + ); + } + + _renderIndexPatternResp() { + if (!this.props.indexPatternResp) { + return null; + } + + return ( + + +

+ +

+ + {this._renderCodeEditor(this.props.indexPatternResp)} + + + ); + } + + _getStatusMsg() { + if (!this.props.importResp || !this.props.importResp.success) { + return i18n.translate('xpack.fileUpload.uploadFailureMsg', { + defaultMessage: 'File upload failed.', + }); + } + + const successMsg = i18n.translate('xpack.fileUpload.uploadSuccessMsg', { + defaultMessage: 'File upload complete: indexed {numFeatures} features.', + values: { + numFeatures: this.props.importResp.docCount, + }, + }); + + const failuredFeaturesMsg = this.props.importResp.failures.length + ? i18n.translate('xpack.fileUpload.failedFeaturesMsg', { + defaultMessage: 'Unable to index {numFailures} features.', + values: { + numFailures: this.props.importResp.failures.length, + }, + }) + : ''; + + return `${successMsg} ${failuredFeaturesMsg}`; + } + + render() { + return ( + + +

{this._getStatusMsg()}

+ + {this._renderIndexResp()} + {this._renderIndexPatternResp()} + +
+ + + + +
+ + + ); + } +} diff --git a/x-pack/plugins/file_upload/public/components/json_import_progress.js b/x-pack/plugins/file_upload/public/components/json_import_progress.js deleted file mode 100644 index 1adf7d9039e5..000000000000 --- a/x-pack/plugins/file_upload/public/components/json_import_progress.js +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { Fragment, Component } from 'react'; -import { i18n } from '@kbn/i18n'; -import { EuiCodeBlock, EuiSpacer, EuiText, EuiTitle, EuiProgress, EuiCallOut } from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { getHttp } from '../kibana_services'; - -export class JsonImportProgress extends Component { - state = { - indexDataJson: null, - indexPatternJson: null, - indexName: '', - importStage: '', - }; - - componentDidUpdate(prevProps, prevState) { - this._setIndex(this.props); - this._formatIndexDataResponse({ ...this.state, ...this.props }); - this._formatIndexPatternResponse({ ...this.state, ...this.props }); - if (prevState.importStage !== this.props.importStage) { - this.setState({ - importStage: this.props.importStage, - }); - } - } - - // Retain last index for UI purposes - _setIndex = ({ indexName }) => { - if (indexName && !this.state.indexName) { - this.setState({ indexName }); - } - }; - - // Format json responses - _formatIndexDataResponse = ({ indexDataResp, indexDataJson }) => { - if (indexDataResp && !indexDataJson) { - this.setState({ indexDataJson: JSON.stringify(indexDataResp, null, 2) }); - } - }; - - _formatIndexPatternResponse = ({ indexPatternResp, indexPatternJson }) => { - if (indexPatternResp && !indexPatternJson) { - this.setState({ indexPatternJson: JSON.stringify(indexPatternResp, null, 2) }); - } - }; - - render() { - const { complete } = this.props; - const { indexPatternJson, indexDataJson, indexName, importStage } = this.state; - const importMessage = complete ? importStage : `${importStage}: ${indexName}`; - - return ( - - {!complete ? : null} - -

- -

- - {importMessage &&

{importMessage}

} - - {complete ? ( - - {indexDataJson ? ( - - -

- -

- - - {indexDataJson} - - - - ) : null} - {indexPatternJson ? ( - - -

- -

- - - {indexPatternJson} - - - - ) : null} - -
- {i18n.translate('xpack.fileUpload.jsonImport.indexModsMsg', { - defaultMessage: 'Further index modifications can be made using\n', - })} - - {i18n.translate('xpack.fileUpload.jsonImport.indexMgmtLink', { - defaultMessage: 'Index Management', - })} - - . -
- - - ) : null} - - ); - } -} diff --git a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js index 660e00ca7f34..d6d1ce615bf0 100644 --- a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js +++ b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js @@ -12,6 +12,7 @@ import PropTypes from 'prop-types'; import { IndexSettings } from './index_settings'; import { getIndexPatternService } from '../kibana_services'; import { GeoJsonFilePicker } from './geojson_file_picker'; +import { ImportCompleteView } from './import_complete_view'; const PHASE = { CONFIGURE: 'CONFIGURE', @@ -38,8 +39,8 @@ export class JsonUploadAndParse extends Component { // Progress-tracking state importStatus: '', phase: PHASE.CONFIGURE, - indexDataResp: '', - indexPatternResp: '', + importResp: undefined, + indexPatternResp: undefined, }; componentDidMount() { @@ -61,21 +62,6 @@ export class JsonUploadAndParse extends Component { } } - _resetFileAndIndexSettings = () => { - if (this._geojsonImporter) { - this._geojsonImporter.destroy(); - this._geojsonImporter = undefined; - } - - this.props.onFileRemove(); - - this.setState({ - indexTypes: [], - selectedIndexType: '', - indexName: '', - }); - }; - _setIndexReady = () => { const isIndexReady = this._geojsonImporter !== undefined && @@ -153,7 +139,7 @@ export class JsonUploadAndParse extends Component { if (!importResp.success) { this.setState({ - indexDataResp: importResp, + importResp, importStatus: i18n.translate('xpack.fileUpload.jsonUploadAndParse.dataIndexingError', { defaultMessage: 'Data indexing error', }), @@ -167,7 +153,7 @@ export class JsonUploadAndParse extends Component { // create index pattern // this.setState({ - indexDataResp: importResp, + importResp, importStatus: i18n.translate('xpack.fileUpload.jsonUploadAndParse.creatingIndexPattern', { defaultMessage: 'Creating index pattern: {indexName}', values: { indexName: this.state.indexName }, @@ -243,6 +229,21 @@ export class JsonUploadAndParse extends Component { ); }; + _onFileClear = () => { + if (this._geojsonImporter) { + this._geojsonImporter.destroy(); + this._geojsonImporter = undefined; + } + + this.props.onFileRemove(); + + this.setState({ + indexTypes: [], + selectedIndexType: '', + indexName: '', + }); + }; + render() { if (this.state.phase === PHASE.IMPORT) { return ( @@ -256,15 +257,17 @@ export class JsonUploadAndParse extends Component { } if (this.state.phase === PHASE.COMPLETE) { - return
complete
; + return ( + + ); } return ( - + Date: Wed, 24 Feb 2021 06:08:05 -0700 Subject: [PATCH 07/26] fix geojson_importer test --- .../public/components/geojson_file_picker.tsx | 2 + .../geojson_importer/geojson_importer.test.js | 124 +++++++----------- .../geojson_importer/geojson_importer.ts | 8 +- .../file_upload/public/importer/index.ts | 1 + 4 files changed, 55 insertions(+), 80 deletions(-) diff --git a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx index 21bc2a1b3446..933556a10183 100644 --- a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx +++ b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx @@ -10,6 +10,7 @@ import { EuiFilePicker, EuiFormRow } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { MB } from '../../common'; import { getMaxBytes, getMaxBytesFormatted } from '../get_max_bytes'; +import { validateFile } from '../importer'; import { GeoJsonImporter, GEOJSON_FILE_TYPES } from '../importer/geojson_importer'; interface Props { @@ -70,6 +71,7 @@ export class GeoJsonFilePicker extends Component { let previewError: string | null = null; let preview: { features: Feature[]; geoFieldTypes: string[]; previewCoverage: number }; try { + validateFile(file, GEOJSON_FILE_TYPES); importer = new GeoJsonImporter(file); preview = await importer.previewFile(10000, MB * 3); if (preview.features.length === 0) { diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js index e348686dc060..b91c23462250 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js @@ -5,7 +5,7 @@ * 2.0. */ -import { GeoJsonImporter } from './geojson_importer'; +import { GeoJsonImporter, toEsDocs } from './geojson_importer'; import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; import '@loaders.gl/polyfills'; @@ -25,9 +25,7 @@ const FEATURE_COLLECTION = { ], }; -describe('readFile', () => { - const setFileProgress = jest.fn((a) => a); - +describe('previewFile', () => { const FILE_WITH_FEATURE_COLLECTION = new File( [JSON.stringify(FEATURE_COLLECTION)], 'testfile.json', @@ -39,38 +37,24 @@ describe('readFile', () => { jest.restoreAllMocks(); }); - test('should throw error if no file provided', async () => { - const importer = new GeoJsonImporter(); - await importer - .readFile(null, setFileProgress, () => { - return true; - }) - .catch((e) => { - expect(e.message).toMatch('Error, no file provided'); - }); - }); - - test('should abort if file parse is cancelled', async () => { - const importer = new GeoJsonImporter(); - - const results = await importer.readFile(FILE_WITH_FEATURE_COLLECTION, setFileProgress, () => { - return false; + test('should stop reading when importer is destroyed', async () => { + const importer = new GeoJsonImporter(FILE_WITH_FEATURE_COLLECTION); + importer.destroy(); + const results = await importer.previewFile(); + expect(results).toEqual({ + features: [], + previewCoverage: 0, + geoFieldTypes: ['geo_shape'], }); - - expect(results).toBeNull(); }); test('should read features from feature collection', async () => { - const importer = new GeoJsonImporter(); - const results = await importer.readFile(FILE_WITH_FEATURE_COLLECTION, setFileProgress, () => { - return true; - }); - - expect(setFileProgress).toHaveBeenCalled(); + const importer = new GeoJsonImporter(FILE_WITH_FEATURE_COLLECTION); + const results = await importer.previewFile(); expect(results).toEqual({ - errors: [], - geometryTypes: ['Point'], - parsedGeojson: FEATURE_COLLECTION, + previewCoverage: 100, + geoFieldTypes: ['geo_point', 'geo_shape'], + features: FEATURE_COLLECTION.features, }); }); @@ -99,20 +83,13 @@ describe('readFile', () => { { type: 'text/json' } ); - const importer = new GeoJsonImporter(); - const results = await importer.readFile( - fileWithFeaturesWithoutGeometry, - setFileProgress, - () => { - return true; - } - ); + const importer = new GeoJsonImporter(fileWithFeaturesWithoutGeometry); + const results = await importer.previewFile(); - expect(setFileProgress).toHaveBeenCalled(); expect(results).toEqual({ - errors: ['2 features without geometry omitted'], - geometryTypes: ['Point'], - parsedGeojson: FEATURE_COLLECTION, + previewCoverage: 100, + geoFieldTypes: ['geo_point', 'geo_shape'], + features: FEATURE_COLLECTION.features, }); }); @@ -134,20 +111,17 @@ describe('readFile', () => { { type: 'text/json' } ); - const importer = new GeoJsonImporter(); - const results = await importer.readFile(fileWithUnwrapedFeature, setFileProgress, () => { - return true; - }); + const importer = new GeoJsonImporter(fileWithUnwrapedFeature); + const results = await importer.previewFile(); - expect(setFileProgress).toHaveBeenCalled(); expect(results).toEqual({ - errors: [], - geometryTypes: ['Point'], - parsedGeojson: FEATURE_COLLECTION, + previewCoverage: 100, + geoFieldTypes: ['geo_point', 'geo_shape'], + features: FEATURE_COLLECTION.features, }); }); - test('should throw if no features', async () => { + test('should return empty feature collection if no features', async () => { const fileWithNoFeatures = new File( [ JSON.stringify({ @@ -159,17 +133,17 @@ describe('readFile', () => { { type: 'text/json' } ); - const importer = new GeoJsonImporter(); - await importer - .readFile(fileWithNoFeatures, setFileProgress, () => { - return true; - }) - .catch((e) => { - expect(e.message).toMatch('Error, no features detected'); - }); + const importer = new GeoJsonImporter(fileWithNoFeatures); + const results = await importer.previewFile(); + + expect(results).toEqual({ + previewCoverage: 100, + geoFieldTypes: ['geo_shape'], + features: [], + }); }); - test('should throw if no features with geometry', async () => { + test('should return empty feature collection if no features with geometry', async () => { const fileWithFeaturesWithNoGeometry = new File( [ JSON.stringify({ @@ -186,22 +160,21 @@ describe('readFile', () => { { type: 'text/json' } ); - const importer = new GeoJsonImporter(); - await importer - .readFile(fileWithFeaturesWithNoGeometry, setFileProgress, () => { - return true; - }) - .catch((e) => { - expect(e.message).toMatch('Error, no features detected'); - }); + const importer = new GeoJsonImporter(fileWithFeaturesWithNoGeometry); + const results = await importer.previewFile(); + + expect(results).toEqual({ + previewCoverage: 100, + geoFieldTypes: ['geo_shape'], + features: [], + }); }); }); -describe('setDocs', () => { +describe('toEsDocs', () => { test('should convert features to geo_point ES documents', () => { - const importer = new GeoJsonImporter(); - importer.setDocs(FEATURE_COLLECTION, ES_FIELD_TYPES.GEO_POINT); - expect(importer.getDocs()).toEqual([ + const esDocs = toEsDocs(FEATURE_COLLECTION.features, ES_FIELD_TYPES.GEO_POINT); + expect(esDocs).toEqual([ { coordinates: [-112.0372, 46.608058], population: 200, @@ -210,9 +183,8 @@ describe('setDocs', () => { }); test('should convert features to geo_shape ES documents', () => { - const importer = new GeoJsonImporter(); - importer.setDocs(FEATURE_COLLECTION, ES_FIELD_TYPES.GEO_SHAPE); - expect(importer.getDocs()).toEqual([ + const esDocs = toEsDocs(FEATURE_COLLECTION.features, ES_FIELD_TYPES.GEO_SHAPE); + expect(esDocs).toEqual([ { coordinates: { type: 'point', diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts index d3e7be8bb42e..d704eca8b9b8 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts @@ -24,7 +24,6 @@ import { callImportRoute, Importer, IMPORT_RETRIES } from '../importer'; import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; // @ts-expect-error import { geoJsonCleanAndValidate } from './geojson_clean_and_validate'; -import { validateFile } from '../validate_file'; import { MB } from '../../../common'; export const GEOJSON_FILE_TYPES = ['.json', '.geojson']; @@ -46,7 +45,6 @@ export class GeoJsonImporter extends Importer { constructor(file: File) { super(); - validateFile(file, GEOJSON_FILE_TYPES); this._file = file; } @@ -61,7 +59,9 @@ export class GeoJsonImporter extends Importer { await this._readUntil(rowLimit, sizeLimit); return { features: [...this._features], - previewCoverage: Math.round((this._unimportedBytesProcessed / this._file.size) * 100), + previewCoverage: this._hasNext + ? Math.round((this._unimportedBytesProcessed / this._file.size) * 100) + : 100, geoFieldTypes: this._geometryTypesMap.has('Point') || this._geometryTypesMap.has('MultiPoint') ? [ES_FIELD_TYPES.GEO_POINT, ES_FIELD_TYPES.GEO_SHAPE] @@ -250,7 +250,7 @@ export class GeoJsonImporter extends Importer { } } -function toEsDocs( +export function toEsDocs( features: Feature[], geoFieldType: ES_FIELD_TYPES.GEO_POINT | ES_FIELD_TYPES.GEO_SHAPE ) { diff --git a/x-pack/plugins/file_upload/public/importer/index.ts b/x-pack/plugins/file_upload/public/importer/index.ts index face822f91ef..22465ae2df8a 100644 --- a/x-pack/plugins/file_upload/public/importer/index.ts +++ b/x-pack/plugins/file_upload/public/importer/index.ts @@ -6,4 +6,5 @@ */ export { importerFactory } from './importer_factory'; +export { validateFile } from './validate_file'; export * from './types'; From 21c38f259bb353e83e1fb91e8d33b8cf2f81edde Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 24 Feb 2021 06:52:46 -0700 Subject: [PATCH 08/26] tslint --- .../public/components/geojson_file_picker.tsx | 26 ++++++++------ .../geojson_importer/geojson_importer.ts | 36 ++++++++++--------- .../public/importer/validate_file.ts | 4 +-- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx index 933556a10183..5776c688d569 100644 --- a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx +++ b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx @@ -6,10 +6,11 @@ */ import React, { Component } from 'react'; +import { Feature } from 'geojson'; import { EuiFilePicker, EuiFormRow } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { MB } from '../../common'; -import { getMaxBytes, getMaxBytesFormatted } from '../get_max_bytes'; +import { getMaxBytesFormatted } from '../get_max_bytes'; import { validateFile } from '../importer'; import { GeoJsonImporter, GEOJSON_FILE_TYPES } from '../importer/geojson_importer'; @@ -22,7 +23,7 @@ interface Props { }: { features: Feature[]; indexName: string; - importer: Importer; + importer: GeoJsonImporter; geoFieldTypes: string[]; }) => void; onClear: () => void; @@ -51,7 +52,7 @@ export class GeoJsonFilePicker extends Component { this._isMounted = false; } - _onFileSelect = async (files: FileList) => { + _onFileSelect = (files: FileList | null) => { this.props.onClear(); this.setState({ @@ -60,16 +61,21 @@ export class GeoJsonFilePicker extends Component { previewSummary: null, }); - if (files.length === 0) { - return; + if (files && files.length) { + this._loadFilePreview(files[0]); } + }; - const file = files[0]; + async _loadFilePreview(file: File) { this.setState({ isLoadingPreview: true }); - let importer: GeoJsonImporter; + let importer: GeoJsonImporter | null = null; let previewError: string | null = null; - let preview: { features: Feature[]; geoFieldTypes: string[]; previewCoverage: number }; + let preview: { + features: Feature[]; + geoFieldTypes: string[]; + previewCoverage: number; + } | null = null; try { validateFile(file, GEOJSON_FILE_TYPES); importer = new GeoJsonImporter(file); @@ -91,7 +97,7 @@ export class GeoJsonFilePicker extends Component { error: previewError, isLoadingPreview: false, previewSummary: - previewError === null + !previewError && preview ? i18n.translate('xpack.fileUpload.geojsonFilePicker.previewSummary', { defaultMessage: 'Previewing {numFeatures} features, {previewCoverage}% of file.', values: { @@ -109,7 +115,7 @@ export class GeoJsonFilePicker extends Component { indexName: file.name.split('.')[0], }); } - }; + } _renderHelpText() { return this.state.previewSummary !== null ? ( diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts index d704eca8b9b8..95c81bfac357 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts @@ -5,10 +5,8 @@ * 2.0. */ -import _ from 'lodash'; import { Feature, - FeatureCollection, Point, MultiPoint, LineString, @@ -19,28 +17,29 @@ import { import { i18n } from '@kbn/i18n'; // @ts-expect-error import { JSONLoader, loadInBatches } from './loaders'; -import { CreateDocsResponse } from '../types'; +import { CreateDocsResponse, ImportResults } from '../types'; import { callImportRoute, Importer, IMPORT_RETRIES } from '../importer'; import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; // @ts-expect-error import { geoJsonCleanAndValidate } from './geojson_clean_and_validate'; -import { MB } from '../../../common'; +import { ImportFailure, ImportResponse, MB } from '../../../common'; export const GEOJSON_FILE_TYPES = ['.json', '.geojson']; export class GeoJsonImporter extends Importer { private _file: File; private _isActive = true; - private _iterator?: Iterator; + private _iterator?: Iterator; private _hasNext = true; private _features: Feature[] = []; private _totalBytesProcessed = 0; private _unimportedBytesProcessed = 0; private _totalFeatures = 0; private _geometryTypesMap = new Map(); - private _invalidCount = 0; + private _invalidFeatures: ImportFailure[] = []; private _prevBatchLastFeature?: Feature; - private _geoFieldType?: ES_FIELD_TYPES.GEO_POINT | ES_FIELD_TYPES.GEO_SHAPE; + private _geoFieldType: ES_FIELD_TYPES.GEO_POINT | ES_FIELD_TYPES.GEO_SHAPE = + ES_FIELD_TYPES.GEO_SHAPE; constructor(file: File) { super(); @@ -55,7 +54,7 @@ export class GeoJsonImporter extends Importer { public async previewFile( rowLimit?: number, sizeLimit?: number - ): { features: Feature[]; geoFieldTypes: string[]; previewCoverage: number } { + ): Promise<{ features: Feature[]; geoFieldTypes: string[]; previewCoverage: number }> { await this._readUntil(rowLimit, sizeLimit); return { features: [...this._features], @@ -93,7 +92,7 @@ export class GeoJsonImporter extends Importer { }; let success = true; - const failures: ImportFailure[] = []; + const failures: ImportFailure[] = [...this._invalidFeatures]; let error; while (this._features.length > 0 || (this._hasNext && this._isActive)) { @@ -144,16 +143,15 @@ export class GeoJsonImporter extends Importer { } } - if (resp.success) { - setImportProgress(progress); - } else { + failures.push(...resp.failures); + + if (!resp.success) { success = false; error = resp.error; - failures.push(...resp.failures); break; } - failures.push(...resp.failures); + setImportProgress(progress); } const result: ImportResults = { @@ -192,7 +190,7 @@ export class GeoJsonImporter extends Importer { }); } - if (!this._isActive) { + if (!this._isActive || !this._iterator) { return; } @@ -231,7 +229,13 @@ export class GeoJsonImporter extends Importer { this._totalFeatures++; if (!rawFeature.geometry || !rawFeature.geometry.type) { - this._invalidCount++; + this._invalidFeatures.push({ + item: this._totalFeatures, + reason: i18n.translate('xpack.fileUpload.geojsonImporter.noGeometry', { + defaultMessage: 'Feature does not contain required field "geometry"', + }), + doc: rawFeature, + }); } else { if (!this._geometryTypesMap.has(rawFeature.geometry.type)) { this._geometryTypesMap.set(rawFeature.geometry.type, true); diff --git a/x-pack/plugins/file_upload/public/importer/validate_file.ts b/x-pack/plugins/file_upload/public/importer/validate_file.ts index c83d55d71db1..98456d98d724 100644 --- a/x-pack/plugins/file_upload/public/importer/validate_file.ts +++ b/x-pack/plugins/file_upload/public/importer/validate_file.ts @@ -43,10 +43,10 @@ export function validateFile(file: File, types: string[]) { } } -function bytesToSize(bytes) { +function bytesToSize(bytes: number) { const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; if (bytes === 0) return 'n/a'; - const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)), 10); + const i = Math.round(Math.floor(Math.log(bytes) / Math.log(1024))); if (i === 0) return `${bytes} ${sizes[i]})`; return `${(bytes / 1024 ** i).toFixed(1)} ${sizes[i]}`; } From e7f686645d8e8041c6b7c751a4919ebbe75d276f Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 24 Feb 2021 06:58:59 -0700 Subject: [PATCH 09/26] i18n fixes --- x-pack/plugins/file_upload/server/plugin.ts | 2 +- .../translations/translations/ja-JP.json | 105 +++++++----------- .../translations/translations/zh-CN.json | 105 +++++++----------- 3 files changed, 83 insertions(+), 129 deletions(-) diff --git a/x-pack/plugins/file_upload/server/plugin.ts b/x-pack/plugins/file_upload/server/plugin.ts index 80021b0ec43e..94b4f0368d56 100644 --- a/x-pack/plugins/file_upload/server/plugin.ts +++ b/x-pack/plugins/file_upload/server/plugin.ts @@ -34,7 +34,7 @@ export class FileUploadPlugin implements Plugin { schema: schema.string(), validation: { regexString: '\\d+[mMgG][bB]', - message: i18n.translate('xpack.ml.maxFileSizeUiSetting.error', { + message: i18n.translate('xpack.fileUpload.maxFileSizeUiSetting.error', { defaultMessage: 'Should be a valid data size. e.g. 200MB, 1GB', }), }, diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 9da5181a776a..047cafc60447 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -2589,15 +2589,6 @@ "indexPatternManagement.actions.deleteButton": "削除", "indexPatternManagement.actions.saveButton": "フィールドを保存", "indexPatternManagement.aliasLabel": "エイリアス", - "indexPatternFieldEditor.color.actions": "アクション", - "indexPatternFieldEditor.color.addColorButton": "色を追加", - "indexPatternFieldEditor.color.backgroundLabel": "背景色", - "indexPatternFieldEditor.color.deleteAria": "削除", - "indexPatternFieldEditor.color.deleteTitle": "色のフォーマットを削除", - "indexPatternFieldEditor.color.exampleLabel": "例", - "indexPatternFieldEditor.color.patternLabel": "パターン（正規表現）", - "indexPatternFieldEditor.color.rangeLabel": "範囲（min:max）", - "indexPatternFieldEditor.color.textColorLabel": "文字の色", "indexPatternManagement.createHeader": "スクリプトフィールドを作成", "indexPatternManagement.createIndexPattern.betaLabel": "ベータ", "indexPatternManagement.createIndexPattern.description": "インデックスパターンは、{single}または{multiple}データソース、{star}と一致します。", @@ -2661,10 +2652,6 @@ "indexPatternManagement.createIndexPatternHeader": "{indexPatternName}の作成", "indexPatternManagement.customLabel": "カスタムラベル", "indexPatternManagement.dataStreamLabel": "データストリーム", - "indexPatternFieldEditor.date.documentationLabel": "ドキュメント", - "indexPatternFieldEditor.date.momentLabel": "Moment.jsのフォーマットパターン（デフォルト: {defaultPattern}）", - "indexPatternFieldEditor.defaultErrorMessage": "このフォーマット構成の使用を試みた際にエラーが発生しました: {message}", - "indexPatternFieldEditor.defaultFormatDropDown": "- デフォルト -", "indexPatternManagement.defaultFormatHeader": "フォーマット (デフォルト: {defaultFormat})", "indexPatternManagement.deleteField.cancelButton": "キャンセル", "indexPatternManagement.deleteField.deleteButton": "削除", @@ -2674,11 +2661,6 @@ "indexPatternManagement.deleteFieldLabel": "削除されたフィールドは復元できません。{separator}続行してよろしいですか？", "indexPatternManagement.disabledCallOutHeader": "スクリプティングが無効です", "indexPatternManagement.disabledCallOutLabel": "Elasticsearchでのすべてのインラインスクリプティングが無効になっています。Kibanaでスクリプトフィールドを使用するには、インラインスクリプティングを有効にする必要があります。", - "indexPatternFieldEditor.duration.decimalPlacesLabel": "小数部分の桁数", - "indexPatternFieldEditor.duration.inputFormatLabel": "インプット形式", - "indexPatternFieldEditor.duration.outputFormatLabel": "アウトプット形式", - "indexPatternFieldEditor.duration.showSuffixLabel": "接尾辞を表示", - "indexPatternFieldEditor.durationErrorMessage": "小数部分の桁数は0から20までの間で指定する必要があります", "indexPatternManagement.editHeader": "{fieldName}を編集", "indexPatternManagement.editIndexPattern.createIndex.defaultButtonDescription": "すべてのデータに完全アグリゲーションを実行", "indexPatternManagement.editIndexPattern.createIndex.defaultButtonText": "標準インデックスパターン", @@ -2794,27 +2776,13 @@ "indexPatternManagement.nameErrorMessage": "名前が必要です", "indexPatternManagement.nameLabel": "名前", "indexPatternManagement.namePlaceholder": "新規スクリプトフィールド", - "indexPatternFieldEditor.number.documentationLabel": "ドキュメント", - "indexPatternFieldEditor.number.numeralLabel": "Numeral.js のフォーマットパターン (デフォルト: {defaultPattern})", "indexPatternManagement.popularityLabel": "利用頻度", - "indexPatternFieldEditor.samples.inputHeader": "インプット", - "indexPatternFieldEditor.samples.outputHeader": "アウトプット", - "indexPatternFieldEditor.samplesHeader": "サンプル", "indexPatternManagement.script.accessWithLabel": "{code} でフィールドにアクセスします。", "indexPatternManagement.script.getHelpLabel": "構文のヒントを得たり、スクリプトの結果をプレビューしたりできます。", "indexPatternManagement.scriptingLanguages.errorFetchingToastDescription": "Elasticsearchから利用可能なスクリプト言語の取得中にエラーが発生しました", "indexPatternManagement.scriptInvalidErrorMessage": "スクリプトが無効です。詳細については、スクリプトプレビューを表示してください", "indexPatternManagement.scriptLabel": "スクリプト", "indexPatternManagement.scriptRequiredErrorMessage": "スクリプトが必要です", - "indexPatternFieldEditor.staticLookup.actions": "アクション", - "indexPatternFieldEditor.staticLookup.addEntryButton": "エントリーを追加", - "indexPatternFieldEditor.staticLookup.deleteAria": "削除", - "indexPatternFieldEditor.staticLookup.deleteTitle": "エントリーの削除", - "indexPatternFieldEditor.staticLookup.keyLabel": "キー", - "indexPatternFieldEditor.staticLookup.leaveBlankPlaceholder": "値をそのままにするには空欄にします", - "indexPatternFieldEditor.staticLookup.unknownKeyLabel": "不明なキーの値", - "indexPatternFieldEditor.staticLookup.valueLabel": "値", - "indexPatternFieldEditor.string.transformLabel": "変換", "indexPatternManagement.syntax.default.formatLabel": "doc['some_field'].value", "indexPatternManagement.syntax.defaultLabel.defaultDetail": "デフォルトで、KibanaのスクリプトフィールドはElasticsearchでの使用を目的に特別に開発されたシンプルでセキュアなスクリプト言語の{painless}を使用します。ドキュメントの値にアクセスするには次のフォーマットを使用します。", "indexPatternManagement.syntax.defaultLabel.painlessLink": "Painless", @@ -2845,8 +2813,48 @@ "indexPatternManagement.testScript.resultsLabel": "最初の10件", "indexPatternManagement.testScript.resultsTitle": "結果を表示", "indexPatternManagement.testScript.submitButtonLabel": "スクリプトを実行", - "indexPatternFieldEditor.truncate.lengthLabel": "フィールドの長さ", "indexPatternManagement.typeLabel": "型", + "indexPatternManagement.warningCallOut.descriptionLabel": "計算値の表示と集約にスクリプトフィールドが使用できます。そのため非常に遅い場合があり、適切に行わないとKibanaが使用できなくなる可能性もあります。この場合安全策はありません。入力ミスがあると、あちこちに予期せぬ例外が起こります！", + "indexPatternManagement.warningCallOutHeader": "十分ご注意ください", + "indexPatternManagement.warningCallOutLabel.callOutDetail": "スクリプトフィールドを使う前に、{scripFields}と{scriptsInAggregation}についてよく理解するようにしてください。", + "indexPatternManagement.warningCallOutLabel.scripFieldsLink": "スクリプトフィールド", + "indexPatternManagement.warningCallOutLabel.scriptsInAggregationLink": "集約におけるスクリプト", + "indexPatternManagement.warningHeader": "廃止警告:", + "indexPatternManagement.warningLabel.painlessLinkLabel": "Painless", + "indexPatternManagement.warningLabel.warningDetail": "{language}は廃止され、KibanaとElasticsearchの次のメジャーバージョンではサポートされなくなります。新規スクリプトフィールドには{painlessLink}を使うことをお勧めします。", + "indexPatternFieldEditor.color.actions": "アクション", + "indexPatternFieldEditor.color.addColorButton": "色を追加", + "indexPatternFieldEditor.color.backgroundLabel": "背景色", + "indexPatternFieldEditor.color.deleteAria": "削除", + "indexPatternFieldEditor.color.deleteTitle": "色のフォーマットを削除", + "indexPatternFieldEditor.color.exampleLabel": "例", + "indexPatternFieldEditor.color.patternLabel": "パターン（正規表現）", + "indexPatternFieldEditor.color.rangeLabel": "範囲（min:max）", + "indexPatternFieldEditor.color.textColorLabel": "文字の色", + "indexPatternFieldEditor.date.documentationLabel": "ドキュメント", + "indexPatternFieldEditor.date.momentLabel": "Moment.jsのフォーマットパターン（デフォルト: {defaultPattern}）", + "indexPatternFieldEditor.defaultErrorMessage": "このフォーマット構成の使用を試みた際にエラーが発生しました: {message}", + "indexPatternFieldEditor.defaultFormatDropDown": "- デフォルト -", + "indexPatternFieldEditor.duration.decimalPlacesLabel": "小数部分の桁数", + "indexPatternFieldEditor.duration.inputFormatLabel": "インプット形式", + "indexPatternFieldEditor.duration.outputFormatLabel": "アウトプット形式", + "indexPatternFieldEditor.duration.showSuffixLabel": "接尾辞を表示", + "indexPatternFieldEditor.durationErrorMessage": "小数部分の桁数は0から20までの間で指定する必要があります", + "indexPatternFieldEditor.number.documentationLabel": "ドキュメント", + "indexPatternFieldEditor.number.numeralLabel": "Numeral.js のフォーマットパターン (デフォルト: {defaultPattern})", + "indexPatternFieldEditor.samples.inputHeader": "インプット", + "indexPatternFieldEditor.samples.outputHeader": "アウトプット", + "indexPatternFieldEditor.samplesHeader": "サンプル", + "indexPatternFieldEditor.staticLookup.actions": "アクション", + "indexPatternFieldEditor.staticLookup.addEntryButton": "エントリーを追加", + "indexPatternFieldEditor.staticLookup.deleteAria": "削除", + "indexPatternFieldEditor.staticLookup.deleteTitle": "エントリーの削除", + "indexPatternFieldEditor.staticLookup.keyLabel": "キー", + "indexPatternFieldEditor.staticLookup.leaveBlankPlaceholder": "値をそのままにするには空欄にします", + "indexPatternFieldEditor.staticLookup.unknownKeyLabel": "不明なキーの値", + "indexPatternFieldEditor.staticLookup.valueLabel": "値", + "indexPatternFieldEditor.string.transformLabel": "変換", + "indexPatternFieldEditor.truncate.lengthLabel": "フィールドの長さ", "indexPatternFieldEditor.url.heightLabel": "高さ", "indexPatternFieldEditor.url.labelTemplateHelpText": "ラベルテンプレートのヘルプ", "indexPatternFieldEditor.url.labelTemplateLabel": "ラベルテンプレート", @@ -2857,14 +2865,6 @@ "indexPatternFieldEditor.url.typeLabel": "型", "indexPatternFieldEditor.url.urlTemplateLabel": "URLテンプレート", "indexPatternFieldEditor.url.widthLabel": "幅", - "indexPatternManagement.warningCallOut.descriptionLabel": "計算値の表示と集約にスクリプトフィールドが使用できます。そのため非常に遅い場合があり、適切に行わないとKibanaが使用できなくなる可能性もあります。この場合安全策はありません。入力ミスがあると、あちこちに予期せぬ例外が起こります！", - "indexPatternManagement.warningCallOutHeader": "十分ご注意ください", - "indexPatternManagement.warningCallOutLabel.callOutDetail": "スクリプトフィールドを使う前に、{scripFields}と{scriptsInAggregation}についてよく理解するようにしてください。", - "indexPatternManagement.warningCallOutLabel.scripFieldsLink": "スクリプトフィールド", - "indexPatternManagement.warningCallOutLabel.scriptsInAggregationLink": "集約におけるスクリプト", - "indexPatternManagement.warningHeader": "廃止警告:", - "indexPatternManagement.warningLabel.painlessLinkLabel": "Painless", - "indexPatternManagement.warningLabel.warningDetail": "{language}は廃止され、KibanaとElasticsearchの次のメジャーバージョンではサポートされなくなります。新規スクリプトフィールドには{painlessLink}を使うことをお勧めします。", "inputControl.control.noIndexPatternTooltip": "index-pattern id が見つかりませんでした: {indexPatternId}.", "inputControl.control.notInitializedTooltip": "コントロールが初期化されていません", "inputControl.control.noValuesDisableTooltip": "「{indexPatternName}」インデックスパターンでいずれのドキュメントにも存在しない「{fieldName}」フィールドがフィルターの対象になっています。異なるフィールドを選択するか、このフィールドに値が入力されているドキュメントをインデックスしてください。", @@ -7437,8 +7437,6 @@ "xpack.features.savedObjectsManagementFeatureName": "保存されたオブジェクトの管理", "xpack.features.visualizeFeatureName": "可視化", "xpack.fileUpload.enterIndexName": "インデックス名を入力", - "xpack.fileUpload.fileParser.noFeaturesDetected": "エラー、機能が検出されませんでした", - "xpack.fileUpload.fileParser.noFileProvided": "エラー、ファイルが提供されていません", "xpack.fileUpload.httpService.fetchError": "フェッチ実行エラー：{error}", "xpack.fileUpload.httpService.noUrl": "URLが指定されていません", "xpack.fileUpload.indexNameReqField": "インデックス名、必須フィールド", @@ -7454,29 +7452,11 @@ "xpack.fileUpload.indexSettings.indexNameContainsIllegalCharactersErrorMessage": "インデックス名に許可されていない文字が含まれています", "xpack.fileUpload.indexSettings.indexNameGuidelines": "インデックス名ガイドライン", "xpack.fileUpload.jsonImport.indexingResponse": "インデックス応答", - "xpack.fileUpload.jsonImport.indexingStatus": "インデックスステータス", "xpack.fileUpload.jsonImport.indexMgmtLink": "インデックス管理", "xpack.fileUpload.jsonImport.indexModsMsg": "次を使用すると、その他のインデックス修正を行うことができます。\n", "xpack.fileUpload.jsonImport.indexPatternResponse": "インデックスパターン応答", - "xpack.fileUpload.jsonIndexFilePicker.acceptableFileSize": "ファイルサイズ {fileSize} は最大ファイルサイズの{maxFileSize} を超えています", - "xpack.fileUpload.jsonIndexFilePicker.acceptableTypesError": "ファイルは使用可能なタイプのいずれかではありません。{types}", - "xpack.fileUpload.jsonIndexFilePicker.coordinateSystemAccepted": "座標は EPSG:4326 座標参照系でなければなりません。", - "xpack.fileUpload.jsonIndexFilePicker.fileParseError": "ファイル解析エラーが検出されました：{error}", - "xpack.fileUpload.jsonIndexFilePicker.filePicker": "ファイルをアップロード", - "xpack.fileUpload.jsonIndexFilePicker.filePickerLabel": "アップロードするファイルを選択", - "xpack.fileUpload.jsonIndexFilePicker.fileProcessingError": "ファイル処理エラー: {errorMessage}", - "xpack.fileUpload.jsonIndexFilePicker.formatsAccepted": "許可されているフォーマット：{acceptedFileTypeStringMessage}", - "xpack.fileUpload.jsonIndexFilePicker.maxSize": "最大サイズ：{maxFileSize}", - "xpack.fileUpload.jsonIndexFilePicker.noFileNameError": "ファイル名が指定されていません", - "xpack.fileUpload.jsonIndexFilePicker.parsingFile": "{featuresProcessed} 機能が解析されました...", - "xpack.fileUpload.jsonIndexFilePicker.unableParseFile": "ファイルをパースできません。{error}", - "xpack.fileUpload.jsonUploadAndParse.creatingIndexPattern": "インデックスパターンを作成中です", "xpack.fileUpload.jsonUploadAndParse.dataIndexingError": "データインデックスエラー", - "xpack.fileUpload.jsonUploadAndParse.dataIndexingStarted": "データインデックスが開始しました", - "xpack.fileUpload.jsonUploadAndParse.indexingComplete": "インデックス完了", - "xpack.fileUpload.jsonUploadAndParse.indexPatternComplete": "インデックスパターンの完了", "xpack.fileUpload.jsonUploadAndParse.indexPatternError": "インデックスパターンエラー", - "xpack.fileUpload.jsonUploadAndParse.writingToIndex": "インデックスに書き込み中", "xpack.fleet.agentBulkActions.agentsSelected": "{count, plural, other {#個のエージェント}}が選択されました", "xpack.fleet.agentBulkActions.clearSelection": "選択した項目をクリア", "xpack.fleet.agentBulkActions.reassignPolicy": "新しいポリシーに割り当てる", @@ -13436,9 +13416,6 @@ "xpack.ml.management.syncSavedObjectsFlyout.sync.error": "一部のジョブを同期できません。", "xpack.ml.management.syncSavedObjectsFlyout.sync.success": "{successCount} {successCount, plural, other {件のジョブ}}が同期されました", "xpack.ml.management.syncSavedObjectsFlyout.syncButton": "同期", - "xpack.ml.maxFileSizeSettingsDescription": "ファイルデータビジュアライザーでデータをインポートするときのファイルサイズ上限を設定します。この設定でサポートされている最大値は1 GBです。", - "xpack.ml.maxFileSizeSettingsError": "200 MB、1 GBなどの有効なデータサイズにしてください。", - "xpack.ml.maxFileSizeSettingsName": "ファイルデータビジュアライザーの最大ファイルアップロードサイズ", "xpack.ml.models.jobService.allOtherRequestsCancelledDescription": " 他のすべてのリクエストはキャンセルされました。", "xpack.ml.models.jobService.categorization.messages.failureToGetTokens": "フィールド値の例のサンプルをトークン化することができませんでした。{message}", "xpack.ml.models.jobService.categorization.messages.insufficientPrivileges": "権限が不十分なため、フィールド値の例のトークン化を実行できませんでした。そのため、フィールド値を確認し、カテゴリー分けジョブでの使用が適当かを確認することができません。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index eaef6eaae30b..0346313cc56f 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -2593,15 +2593,6 @@ "indexPatternManagement.actions.deleteButton": "删除", "indexPatternManagement.actions.saveButton": "保存字段", "indexPatternManagement.aliasLabel": "别名", - "indexPatternFieldEditor.color.actions": "操作", - "indexPatternFieldEditor.color.addColorButton": "添加颜色", - "indexPatternFieldEditor.color.backgroundLabel": "背景色", - "indexPatternFieldEditor.color.deleteAria": "删除", - "indexPatternFieldEditor.color.deleteTitle": "删除颜色格式", - "indexPatternFieldEditor.color.exampleLabel": "示例", - "indexPatternFieldEditor.color.patternLabel": "模式（正则表达式）", - "indexPatternFieldEditor.color.rangeLabel": "范围（最小值:最大值）", - "indexPatternFieldEditor.color.textColorLabel": "文本颜色", "indexPatternManagement.createHeader": "创建脚本字段", "indexPatternManagement.createIndexPattern.betaLabel": "公测版", "indexPatternManagement.createIndexPattern.description": "索引模式可以匹配单个源，例如 {single} 或 {multiple} 个数据源、{star}。", @@ -2665,10 +2656,6 @@ "indexPatternManagement.createIndexPatternHeader": "创建 {indexPatternName}", "indexPatternManagement.customLabel": "定制标签", "indexPatternManagement.dataStreamLabel": "数据流", - "indexPatternFieldEditor.date.documentationLabel": "文档", - "indexPatternFieldEditor.date.momentLabel": "Moment.js 格式模式（默认值：{defaultPattern}）", - "indexPatternFieldEditor.defaultErrorMessage": "尝试使用此格式配置时发生错误：{message}", - "indexPatternFieldEditor.defaultFormatDropDown": "- 默认值 -", "indexPatternManagement.defaultFormatHeader": "格式（默认值：{defaultFormat}）", "indexPatternManagement.deleteField.cancelButton": "取消", "indexPatternManagement.deleteField.deleteButton": "删除", @@ -2678,11 +2665,6 @@ "indexPatternManagement.deleteFieldLabel": "您无法恢复已删除字段。{separator}确定要执行此操作？", "indexPatternManagement.disabledCallOutHeader": "脚本已禁用", "indexPatternManagement.disabledCallOutLabel": "所有内联脚本在 Elasticsearch 中已禁用。必须至少为一种语言启用内联脚本，才能在 Kibana 中使用脚本字段。", - "indexPatternFieldEditor.duration.decimalPlacesLabel": "小数位数", - "indexPatternFieldEditor.duration.inputFormatLabel": "输入格式", - "indexPatternFieldEditor.duration.outputFormatLabel": "输出格式", - "indexPatternFieldEditor.duration.showSuffixLabel": "显示后缀", - "indexPatternFieldEditor.durationErrorMessage": "小数位数必须介于 0 和 20 之间", "indexPatternManagement.editHeader": "编辑 {fieldName}", "indexPatternManagement.editIndexPattern.createIndex.defaultButtonDescription": "对任何数据执行完全聚合", "indexPatternManagement.editIndexPattern.createIndex.defaultButtonText": "标准索引模式", @@ -2798,27 +2780,13 @@ "indexPatternManagement.nameErrorMessage": "“名称”必填", "indexPatternManagement.nameLabel": "名称", "indexPatternManagement.namePlaceholder": "新建脚本字段", - "indexPatternFieldEditor.number.documentationLabel": "文档", - "indexPatternFieldEditor.number.numeralLabel": "Numeral.js 格式模式（默认值：{defaultPattern}）", "indexPatternManagement.popularityLabel": "常见度", - "indexPatternFieldEditor.samples.inputHeader": "输入", - "indexPatternFieldEditor.samples.outputHeader": "输出", - "indexPatternFieldEditor.samplesHeader": "样例", "indexPatternManagement.script.accessWithLabel": "使用 {code} 访问字段。", "indexPatternManagement.script.getHelpLabel": "获取该语法的帮助，预览脚本的结果。", "indexPatternManagement.scriptingLanguages.errorFetchingToastDescription": "从 Elasticsearch 获取可用的脚本语言时出错", "indexPatternManagement.scriptInvalidErrorMessage": "脚本无效。查看脚本预览以了解详情", "indexPatternManagement.scriptLabel": "脚本", "indexPatternManagement.scriptRequiredErrorMessage": "“脚本”必填", - "indexPatternFieldEditor.staticLookup.actions": "操作", - "indexPatternFieldEditor.staticLookup.addEntryButton": "添加条目", - "indexPatternFieldEditor.staticLookup.deleteAria": "删除", - "indexPatternFieldEditor.staticLookup.deleteTitle": "删除条目", - "indexPatternFieldEditor.staticLookup.keyLabel": "键", - "indexPatternFieldEditor.staticLookup.leaveBlankPlaceholder": "留空可使值保持原样", - "indexPatternFieldEditor.staticLookup.unknownKeyLabel": "未知键的值", - "indexPatternFieldEditor.staticLookup.valueLabel": "值", - "indexPatternFieldEditor.string.transformLabel": "转换", "indexPatternManagement.syntax.default.formatLabel": "doc['some_field'].value", "indexPatternManagement.syntax.defaultLabel.defaultDetail": "默认情况下，Kibana 脚本字段使用 {painless}（一种简单且安全的脚本语言，专用于 Elasticsearch）通过以下格式访问文档中的值：", "indexPatternManagement.syntax.defaultLabel.painlessLink": "Painless", @@ -2849,8 +2817,48 @@ "indexPatternManagement.testScript.resultsLabel": "前 10 个结果", "indexPatternManagement.testScript.resultsTitle": "预览结果", "indexPatternManagement.testScript.submitButtonLabel": "运行脚本", - "indexPatternFieldEditor.truncate.lengthLabel": "字段长度", "indexPatternManagement.typeLabel": "类型", + "indexPatternManagement.warningCallOut.descriptionLabel": "脚本字段可用于显示并聚合计算值。因此，它们会很慢，如果操作不当，会导致 Kibana 不可用。此处没有安全网。如果拼写错误，则在任何地方都会引发异常！", + "indexPatternManagement.warningCallOutHeader": "谨慎操作", + "indexPatternManagement.warningCallOutLabel.callOutDetail": "请先熟悉{scripFields}以及{scriptsInAggregation}，然后再使用脚本字段。", + "indexPatternManagement.warningCallOutLabel.scripFieldsLink": "脚本字段", + "indexPatternManagement.warningCallOutLabel.scriptsInAggregationLink": "聚合中的脚本", + "indexPatternManagement.warningHeader": "过时警告：", + "indexPatternManagement.warningLabel.painlessLinkLabel": "Painless", + "indexPatternManagement.warningLabel.warningDetail": "{language} 已过时，Kibana 和 Elasticsearch 下一主要版本将移除支持。建议将 {painlessLink} 用于新的脚本字段。", + "indexPatternFieldEditor.color.actions": "操作", + "indexPatternFieldEditor.color.addColorButton": "添加颜色", + "indexPatternFieldEditor.color.backgroundLabel": "背景色", + "indexPatternFieldEditor.color.deleteAria": "删除", + "indexPatternFieldEditor.color.deleteTitle": "删除颜色格式", + "indexPatternFieldEditor.color.exampleLabel": "示例", + "indexPatternFieldEditor.color.patternLabel": "模式（正则表达式）", + "indexPatternFieldEditor.color.rangeLabel": "范围（最小值:最大值）", + "indexPatternFieldEditor.color.textColorLabel": "文本颜色", + "indexPatternFieldEditor.date.documentationLabel": "文档", + "indexPatternFieldEditor.date.momentLabel": "Moment.js 格式模式（默认值：{defaultPattern}）", + "indexPatternFieldEditor.defaultErrorMessage": "尝试使用此格式配置时发生错误：{message}", + "indexPatternFieldEditor.defaultFormatDropDown": "- 默认值 -", + "indexPatternFieldEditor.duration.decimalPlacesLabel": "小数位数", + "indexPatternFieldEditor.duration.inputFormatLabel": "输入格式", + "indexPatternFieldEditor.duration.outputFormatLabel": "输出格式", + "indexPatternFieldEditor.duration.showSuffixLabel": "显示后缀", + "indexPatternFieldEditor.durationErrorMessage": "小数位数必须介于 0 和 20 之间", + "indexPatternFieldEditor.number.documentationLabel": "文档", + "indexPatternFieldEditor.number.numeralLabel": "Numeral.js 格式模式（默认值：{defaultPattern}）", + "indexPatternFieldEditor.samples.inputHeader": "输入", + "indexPatternFieldEditor.samples.outputHeader": "输出", + "indexPatternFieldEditor.samplesHeader": "样例", + "indexPatternFieldEditor.staticLookup.actions": "操作", + "indexPatternFieldEditor.staticLookup.addEntryButton": "添加条目", + "indexPatternFieldEditor.staticLookup.deleteAria": "删除", + "indexPatternFieldEditor.staticLookup.deleteTitle": "删除条目", + "indexPatternFieldEditor.staticLookup.keyLabel": "键", + "indexPatternFieldEditor.staticLookup.leaveBlankPlaceholder": "留空可使值保持原样", + "indexPatternFieldEditor.staticLookup.unknownKeyLabel": "未知键的值", + "indexPatternFieldEditor.staticLookup.valueLabel": "值", + "indexPatternFieldEditor.string.transformLabel": "转换", + "indexPatternFieldEditor.truncate.lengthLabel": "字段长度", "indexPatternFieldEditor.url.heightLabel": "高", "indexPatternFieldEditor.url.labelTemplateHelpText": "标签模板帮助", "indexPatternFieldEditor.url.labelTemplateLabel": "标签模板", @@ -2861,14 +2869,6 @@ "indexPatternFieldEditor.url.typeLabel": "类型", "indexPatternFieldEditor.url.urlTemplateLabel": "URL 模板", "indexPatternFieldEditor.url.widthLabel": "宽", - "indexPatternManagement.warningCallOut.descriptionLabel": "脚本字段可用于显示并聚合计算值。因此，它们会很慢，如果操作不当，会导致 Kibana 不可用。此处没有安全网。如果拼写错误，则在任何地方都会引发异常！", - "indexPatternManagement.warningCallOutHeader": "谨慎操作", - "indexPatternManagement.warningCallOutLabel.callOutDetail": "请先熟悉{scripFields}以及{scriptsInAggregation}，然后再使用脚本字段。", - "indexPatternManagement.warningCallOutLabel.scripFieldsLink": "脚本字段", - "indexPatternManagement.warningCallOutLabel.scriptsInAggregationLink": "聚合中的脚本", - "indexPatternManagement.warningHeader": "过时警告：", - "indexPatternManagement.warningLabel.painlessLinkLabel": "Painless", - "indexPatternManagement.warningLabel.warningDetail": "{language} 已过时，Kibana 和 Elasticsearch 下一主要版本将移除支持。建议将 {painlessLink} 用于新的脚本字段。", "inputControl.control.noIndexPatternTooltip": "找不到索引模式 ID：{indexPatternId}。", "inputControl.control.notInitializedTooltip": "尚未初始化控件", "inputControl.control.noValuesDisableTooltip": "按 “{fieldName}” 字段进行了筛选，但 “{indexPatternName}” 索引模式中的任何文档上都不存在该字段。选择不同的字段或索引包含此字段的值的文档。", @@ -7456,8 +7456,6 @@ "xpack.features.savedObjectsManagementFeatureName": "已保存对象管理", "xpack.features.visualizeFeatureName": "Visualize", "xpack.fileUpload.enterIndexName": "输入索引名称", - "xpack.fileUpload.fileParser.noFeaturesDetected": "错误，未检测到特征", - "xpack.fileUpload.fileParser.noFileProvided": "错误，未提供任何文件", "xpack.fileUpload.httpService.fetchError": "执行提取时出错：{error}", "xpack.fileUpload.httpService.noUrl": "未提供 URL", "xpack.fileUpload.indexNameReqField": "索引名称，必填字段", @@ -7473,29 +7471,11 @@ "xpack.fileUpload.indexSettings.indexNameContainsIllegalCharactersErrorMessage": "索引名称包含非法字符。", "xpack.fileUpload.indexSettings.indexNameGuidelines": "索引名称指引", "xpack.fileUpload.jsonImport.indexingResponse": "索引响应", - "xpack.fileUpload.jsonImport.indexingStatus": "索引状态", "xpack.fileUpload.jsonImport.indexMgmtLink": "索引管理", "xpack.fileUpload.jsonImport.indexModsMsg": "要进一步做索引修改，可以使用\n", "xpack.fileUpload.jsonImport.indexPatternResponse": "索引模式响应", - "xpack.fileUpload.jsonIndexFilePicker.acceptableFileSize": "文件大小 {fileSize} 超过最大文件大小 {maxFileSize}", - "xpack.fileUpload.jsonIndexFilePicker.acceptableTypesError": "文件不是可接受类型之一：{types}", - "xpack.fileUpload.jsonIndexFilePicker.coordinateSystemAccepted": "坐标必须在 EPSG:4326 坐标参考系中。", - "xpack.fileUpload.jsonIndexFilePicker.fileParseError": "检测到文件解析错误：{error}", - "xpack.fileUpload.jsonIndexFilePicker.filePicker": "上传文件", - "xpack.fileUpload.jsonIndexFilePicker.filePickerLabel": "选择文件进行上传", - "xpack.fileUpload.jsonIndexFilePicker.fileProcessingError": "文件处理错误：{errorMessage}", - "xpack.fileUpload.jsonIndexFilePicker.formatsAccepted": "接受的格式：{acceptedFileTypeStringMessage}", - "xpack.fileUpload.jsonIndexFilePicker.maxSize": "最大大小：{maxFileSize}", - "xpack.fileUpload.jsonIndexFilePicker.noFileNameError": "未提供任何文件名称", - "xpack.fileUpload.jsonIndexFilePicker.parsingFile": "已处理 {featuresProcessed} 个特征......", - "xpack.fileUpload.jsonIndexFilePicker.unableParseFile": "无法解析文件：{error}", - "xpack.fileUpload.jsonUploadAndParse.creatingIndexPattern": "正在创建索引模式", "xpack.fileUpload.jsonUploadAndParse.dataIndexingError": "数据索引错误", - "xpack.fileUpload.jsonUploadAndParse.dataIndexingStarted": "数据索引已启动", - "xpack.fileUpload.jsonUploadAndParse.indexingComplete": "索引完成", - "xpack.fileUpload.jsonUploadAndParse.indexPatternComplete": "索引模式完成", "xpack.fileUpload.jsonUploadAndParse.indexPatternError": "索引模式错误", - "xpack.fileUpload.jsonUploadAndParse.writingToIndex": "正在写入索引", "xpack.fleet.agentBulkActions.agentsSelected": "已选择 {count, plural, other {# 个代理}}", "xpack.fleet.agentBulkActions.clearSelection": "清除所选内容", "xpack.fleet.agentBulkActions.reassignPolicy": "分配到新策略", @@ -13468,9 +13448,6 @@ "xpack.ml.management.syncSavedObjectsFlyout.sync.error": "一些作业无法同步。", "xpack.ml.management.syncSavedObjectsFlyout.sync.success": "{successCount} 个{successCount, plural, other {作业}}已同步", "xpack.ml.management.syncSavedObjectsFlyout.syncButton": "同步", - "xpack.ml.maxFileSizeSettingsDescription": "设置在文件数据可视化工具中导入数据时的文件大小限制。此设置支持的最高值为 1GB。", - "xpack.ml.maxFileSizeSettingsError": "应为有效的数据大小。如 200MB、1GB", - "xpack.ml.maxFileSizeSettingsName": "文件数据可视化工具最大文件上传大小", "xpack.ml.models.jobService.allOtherRequestsCancelledDescription": " 所有其他请求已取消。", "xpack.ml.models.jobService.categorization.messages.failureToGetTokens": "无法对示例字段值样本进行分词。{message}", "xpack.ml.models.jobService.categorization.messages.insufficientPrivileges": "由于权限不足，无法对字段值示例执行分词。因此，无法检查字段值是否适合用于归类作业。", From 07ff940b133d28688182e0b4f06457f109e04d3a Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 24 Feb 2021 07:22:06 -0700 Subject: [PATCH 10/26] cleanup --- x-pack/plugins/file_upload/common/constants.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/file_upload/common/constants.ts b/x-pack/plugins/file_upload/common/constants.ts index f64626b5a70a..ea36e5146670 100644 --- a/x-pack/plugins/file_upload/common/constants.ts +++ b/x-pack/plugins/file_upload/common/constants.ts @@ -6,7 +6,6 @@ */ export const UI_SETTING_MAX_FILE_SIZE = 'fileUpload:maxFileSize'; -// export const FILE_DATA_VISUALIZER_MAX_FILE_SIZE = 'ml:fileDataVisualizerMaxFileSize'; export const MB = Math.pow(2, 20); export const MAX_FILE_SIZE = '100MB'; From ed7a0fa9d9cecabecef3ea80c4efe5f5f7411175 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 24 Feb 2021 07:38:45 -0700 Subject: [PATCH 11/26] update documenation for advanced setting rename --- docs/management/advanced-options.asciidoc | 10 ++++---- docs/user/ml/index.asciidoc | 28 +++++++++++------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/management/advanced-options.asciidoc b/docs/management/advanced-options.asciidoc index c7d5242da69d..3ee7a0471eec 100644 --- a/docs/management/advanced-options.asciidoc +++ b/docs/management/advanced-options.asciidoc @@ -83,6 +83,10 @@ specific dashboard, application, or saved object as they enter each space. [[fields-popularlimit]]`fields:popularLimit`:: The top N most popular fields to show. +[[fileupload-maxfilesize]]`fileUpload:maxFileSize`:: +Sets the file size limit when importing files. The default +value is `100MB`. The highest supported value for this setting is `1GB`. + [[filtereditor-suggestvalues]]`filterEditor:suggestValues`:: Set this property to `false` to prevent the filter editor from suggesting values for fields. @@ -257,7 +261,7 @@ Hides the "Time" column in *Discover* and in all saved searches on dashboards. Highlights results in *Discover* and saved searches on dashboards. Highlighting slows requests when working on big documents. -[[doctable-legacy]]`doc_table:legacy`:: +[[doctable-legacy]]`doc_table:legacy`:: Controls the way the document table looks and works. Set this property to `true` to revert to the legacy implementation. [[discover-searchFieldsFromSource]]`discover:searchFieldsFromSource`:: @@ -281,10 +285,6 @@ must contain `from` and `to` values (see {ref}/common-options.html#date-math[accepted formats]). It is ignored unless `ml:anomalyDetection:results:enableTimeDefaults` is enabled. -[[ml-filedatavisualizermaxfilesize]]`ml:fileDataVisualizerMaxFileSize`:: -Sets the file size limit when importing data in the {data-viz}. The default -value is `100MB`. The highest supported value for this setting is `1GB`. - [float] [[kibana-notification-settings]] diff --git a/docs/user/ml/index.asciidoc b/docs/user/ml/index.asciidoc index fa15e0652e2a..3c463da842fa 100644 --- a/docs/user/ml/index.asciidoc +++ b/docs/user/ml/index.asciidoc @@ -20,10 +20,10 @@ image::user/ml/images/ml-data-visualizer-sample.jpg[{data-viz} for sample flight experimental[] You can also upload a CSV, NDJSON, or log file. The *{data-viz}* identifies the file format and field mappings. You can then optionally import that data into an {es} index. To change the default file size limit, see -<>. +<>. If {stack-security-features} are enabled, users must have the necessary -privileges to use {ml-features}. Refer to +privileges to use {ml-features}. Refer to {ml-docs}/setup.html#setup-privileges[Set up {ml-features}]. NOTE: There are limitations in {ml-features} that affect {kib}. For more information, refer to {ml-docs}/ml-limitations.html[Machine learning]. @@ -40,15 +40,15 @@ false positives. {anomaly-detect-cap} runs in and scales with {es}, and includes an intuitive UI on the {kib} *Machine Learning* page for creating {anomaly-jobs} and understanding results. -If you have a license that includes the {ml-features}, you can +If you have a license that includes the {ml-features}, you can create {anomaly-jobs} and manage jobs and {dfeeds} from the *Job Management* -pane: +pane: [role="screenshot"] image::user/ml/images/ml-job-management.png[Job Management] -You can use the *Settings* pane to create and edit -{ml-docs}/ml-calendars.html[calendars] and the filters that are used in +You can use the *Settings* pane to create and edit +{ml-docs}/ml-calendars.html[calendars] and the filters that are used in {ml-docs}/ml-rules.html[custom rules]: [role="screenshot"] @@ -69,13 +69,13 @@ occurring in your operational environment at that time: image::user/ml/images/ml-annotations-list.png[Single Metric Viewer with annotations] In some circumstances, annotations are also added automatically. For example, if -the {anomaly-job} detects that there is missing data, it annotates the affected -time period. For more information, see -{ml-docs}/ml-delayed-data-detection.html[Handling delayed data]. The +the {anomaly-job} detects that there is missing data, it annotates the affected +time period. For more information, see +{ml-docs}/ml-delayed-data-detection.html[Handling delayed data]. The *Job Management* pane shows the full list of annotations for each job. -NOTE: The {kib} {ml-features} use pop-ups. You must configure your web -browser so that it does not block pop-up windows or create an exception for your +NOTE: The {kib} {ml-features} use pop-ups. You must configure your web +browser so that it does not block pop-up windows or create an exception for your {kib} URL. For more information about the {anomaly-detect} feature, see @@ -89,7 +89,7 @@ experimental[] The Elastic {ml} {dfanalytics} feature enables you to analyze your data using {classification}, {oldetection}, and {regression} algorithms and generate new -indices that contain the results alongside your source data. +indices that contain the results alongside your source data. If you have a license that includes the {ml-features}, you can create {dfanalytics-jobs} and view their results on the *Data Frame Analytics* page in @@ -98,5 +98,5 @@ If you have a license that includes the {ml-features}, you can create [role="screenshot"] image::user/ml/images/outliers.png[{oldetection-cap} results in {kib}] -For more information about the {dfanalytics} feature, see -{ml-docs}/ml-dfanalytics.html[{ml-cap} {dfanalytics}]. \ No newline at end of file +For more information about the {dfanalytics} feature, see +{ml-docs}/ml-dfanalytics.html[{ml-cap} {dfanalytics}]. From 295ef6c2d5bd455c4c5a9069b3788229d275ffcd Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 24 Feb 2021 09:35:37 -0700 Subject: [PATCH 12/26] advanced settings usage_collection --- .../server/collectors/management/schema.ts | 1 + .../server/collectors/management/types.ts | 1 + src/plugins/telemetry/schema/oss_plugins.json | 3 +++ .../schema/xpack_plugins.json | 26 +++++++++---------- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index d632c3ad61a8..7abf0eef7d60 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -73,6 +73,7 @@ export const stackManagementSchema: MakeSchemaFrom = { 'discover:sort:defaultOrder': { type: 'keyword' }, 'context:step': { type: 'long' }, 'accessibility:disableAnimations': { type: 'boolean' }, + 'fileUpload:maxFileSize': { type: 'keyword' }, 'ml:fileDataVisualizerMaxFileSize': { type: 'keyword' }, 'ml:anomalyDetection:results:enableTimeDefaults': { type: 'boolean' }, 'ml:anomalyDetection:results:timeDefaults': { type: 'keyword' }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 698e7e711529..91c4b7bce032 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -73,6 +73,7 @@ export interface UsageStats { 'discover:sort:defaultOrder': string; 'context:step': number; 'accessibility:disableAnimations': boolean; + 'fileUpload:maxFileSize': string; 'ml:fileDataVisualizerMaxFileSize': string; 'ml:anomalyDetection:results:enableTimeDefaults': boolean; 'ml:anomalyDetection:results:timeDefaults': string; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 566d10182b54..1d670ac4f43b 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -4235,6 +4235,9 @@ "accessibility:disableAnimations": { "type": "boolean" }, + "fileUpload:maxFileSize": { + "type": "keyword" + }, "ml:fileDataVisualizerMaxFileSize": { "type": "keyword" }, diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index d3487078fd11..db7f22945a0d 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -1656,6 +1656,19 @@ } } }, + "search-session": { + "properties": { + "transientCount": { + "type": "long" + }, + "persistedCount": { + "type": "long" + }, + "totalCount": { + "type": "long" + } + } + }, "discoverEnhanced": { "properties": { "exploreDataInChartActionEnabled": { @@ -3183,19 +3196,6 @@ } } }, - "search-session": { - "properties": { - "transientCount": { - "type": "long" - }, - "persistedCount": { - "type": "long" - }, - "totalCount": { - "type": "long" - } - } - }, "security_solution": { "properties": { "detections": { From 5035cc1df6b8ffaba94019e5d68b30c74a41fc67 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Thu, 25 Feb 2021 16:13:58 -0700 Subject: [PATCH 13/26] remove ml:fileDataVisualizerMaxFileSize from schemas and types --- .../server/collectors/management/schema.ts | 1 - .../server/collectors/management/types.ts | 1 - src/plugins/telemetry/schema/oss_plugins.json | 3 --- 3 files changed, 5 deletions(-) diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index 7abf0eef7d60..19a1b6ac5f5d 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -74,7 +74,6 @@ export const stackManagementSchema: MakeSchemaFrom = { 'context:step': { type: 'long' }, 'accessibility:disableAnimations': { type: 'boolean' }, 'fileUpload:maxFileSize': { type: 'keyword' }, - 'ml:fileDataVisualizerMaxFileSize': { type: 'keyword' }, 'ml:anomalyDetection:results:enableTimeDefaults': { type: 'boolean' }, 'ml:anomalyDetection:results:timeDefaults': { type: 'keyword' }, 'truncate:maxHeight': { type: 'long' }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 91c4b7bce032..8bbc14e0678d 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -74,7 +74,6 @@ export interface UsageStats { 'context:step': number; 'accessibility:disableAnimations': boolean; 'fileUpload:maxFileSize': string; - 'ml:fileDataVisualizerMaxFileSize': string; 'ml:anomalyDetection:results:enableTimeDefaults': boolean; 'ml:anomalyDetection:results:timeDefaults': string; 'truncate:maxHeight': number; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 1d670ac4f43b..70a6968d4128 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -4238,9 +4238,6 @@ "fileUpload:maxFileSize": { "type": "keyword" }, - "ml:fileDataVisualizerMaxFileSize": { - "type": "keyword" - }, "ml:anomalyDetection:results:enableTimeDefaults": { "type": "boolean" }, From a15ca72ef5fb507c11dcdcc40f9fa9b5e4fc3737 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Sat, 27 Feb 2021 09:30:21 -0700 Subject: [PATCH 14/26] add copy buttons for import response and fix geojson upload functional tests --- .../components/import_complete_view.tsx | 130 ++++++++++-------- .../test/functional/page_objects/gis_page.ts | 24 ++-- 2 files changed, 85 insertions(+), 69 deletions(-) diff --git a/x-pack/plugins/file_upload/public/components/import_complete_view.tsx b/x-pack/plugins/file_upload/public/components/import_complete_view.tsx index 0a9fd5373751..c393e2d4bb0a 100644 --- a/x-pack/plugins/file_upload/public/components/import_complete_view.tsx +++ b/x-pack/plugins/file_upload/public/components/import_complete_view.tsx @@ -7,7 +7,16 @@ import React, { Component, Fragment } from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiCallOut, EuiSpacer, EuiText, EuiTitle } from '@elastic/eui'; +import { + EuiButtonIcon, + EuiCallOut, + EuiCopy, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, + EuiText, + EuiTitle, +} from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { CodeEditor, KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public'; import { getHttp, getUiSettings } from '../kibana_services'; @@ -19,71 +28,62 @@ const services = { interface Props { importResp?: ImportResponse; - indexPatternResp?: unknown; + indexPatternResp?: object; } export class ImportCompleteView extends Component { - _renderCodeEditor(value: unknown) { - return ( -
- {}} - options={{ - readOnly: true, - lineNumbers: 'off', - fontSize: 12, - minimap: { - enabled: false, - }, - scrollBeyondLastLine: false, - wordWrap: 'on', - wrappingIndent: 'indent', - automaticLayout: true, - }} - /> -
- ); - } - - _renderIndexResp() { - if (!this.props.importResp) { + _renderCodeEditor(json: object | undefined, title: string, copyButtonDataTestSubj: string) { + if (!json) { return null; } - return ( - - -

- -

- - {this._renderCodeEditor(this.props.importResp)} - - - ); - } - - _renderIndexPatternResp() { - if (!this.props.indexPatternResp) { - return null; - } + const jsonAsString = JSON.stringify(json, null, 2); return ( - -

- -

- - {this._renderCodeEditor(this.props.indexPatternResp)} + + + +

{title}

+ + + + + + {(copy) => ( + + )} + + + +
+ {}} + options={{ + readOnly: true, + lineNumbers: 'off', + fontSize: 12, + minimap: { + enabled: false, + }, + scrollBeyondLastLine: false, + wordWrap: 'on', + wrappingIndent: 'indent', + automaticLayout: true, + }} + /> +
); @@ -121,8 +121,20 @@ export class ImportCompleteView extends Component {

{this._getStatusMsg()}

 - {this._renderIndexResp()} - {this._renderIndexPatternResp()} + {this._renderCodeEditor( + this.props.importResp, + i18n.translate('xpack.fileUpload.jsonImport.indexingResponse', { + defaultMessage: 'Import response', + }), + 'indexRespCopyButton' + )} + {this._renderCodeEditor( + this.props.indexPatternResp, + i18n.translate('xpack.fileUpload.jsonImport.indexPatternResponse', { + defaultMessage: 'Index pattern response', + }), + 'indexPatternRespCopyButton' + )}
{ layerAddReady = await this.importFileButtonEnabled(); @@ -454,21 +454,25 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte ); } - async getCodeBlockParsedJson(dataTestSubjName: string) { - log.debug(`Get parsed code block for ${dataTestSubjName}`); - const indexRespCodeBlock = await testSubjects.find(`${dataTestSubjName}`); - const indexRespJson = await indexRespCodeBlock.getAttribute('innerText'); - return JSON.parse(indexRespJson); + async clickCopyButton(dataTestSubj: string): Promise { + log.debug(`Click ${dataTestSubj} copy button`); + + // The "clipboard-read" permission of the Permissions API must be granted + if (!(await browser.checkBrowserPermission('clipboard-read'))) { + return; + } + + await testSubjects.click(dataTestSubj); + + return await browser.getClipboardValue(); } async getIndexResults() { - log.debug('Get index results'); - return await this.getCodeBlockParsedJson('indexRespCodeBlock'); + return JSON.parse(await this.clickCopyButton('indexRespCopyButton')); } async getIndexPatternResults() { - log.debug('Get index pattern results'); - return await this.getCodeBlockParsedJson('indexPatternRespCodeBlock'); + return JSON.parse(await this.clickCopyButton('indexPatternRespCopyButton')); } async setLayerQuery(layerName: string, query: string) { From b2e03fbc855ff2679b351d0ea874411d6afbe55c Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Sat, 27 Feb 2021 14:51:20 -0700 Subject: [PATCH 15/26] tslint --- x-pack/test/functional/page_objects/gis_page.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/functional/page_objects/gis_page.ts b/x-pack/test/functional/page_objects/gis_page.ts index 5a3ded3ca944..8939c035b47f 100644 --- a/x-pack/test/functional/page_objects/gis_page.ts +++ b/x-pack/test/functional/page_objects/gis_page.ts @@ -459,7 +459,7 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte // The "clipboard-read" permission of the Permissions API must be granted if (!(await browser.checkBrowserPermission('clipboard-read'))) { - return; + throw new Error(`Unable to get browser clipboard read permission`); } await testSubjects.click(dataTestSubj); From f48dc87f05a10fdb2aff6f0688a76c044afe92f4 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Sat, 27 Feb 2021 16:52:53 -0700 Subject: [PATCH 16/26] remove clipboard-read check --- x-pack/test/functional/page_objects/gis_page.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/test/functional/page_objects/gis_page.ts b/x-pack/test/functional/page_objects/gis_page.ts index 8939c035b47f..6694a494cf85 100644 --- a/x-pack/test/functional/page_objects/gis_page.ts +++ b/x-pack/test/functional/page_objects/gis_page.ts @@ -457,11 +457,6 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte async clickCopyButton(dataTestSubj: string): Promise { log.debug(`Click ${dataTestSubj} copy button`); - // The "clipboard-read" permission of the Permissions API must be granted - if (!(await browser.checkBrowserPermission('clipboard-read'))) { - throw new Error(`Unable to get browser clipboard read permission`); - } - await testSubjects.click(dataTestSubj); return await browser.getClipboardValue(); From be10ff7730a70f892b633218d11c4428fdd7b91b Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Sun, 28 Feb 2021 12:01:01 -0700 Subject: [PATCH 17/26] return early if env does not support reading from clipboard --- .../apps/maps/import_geojson/file_indexing_panel.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/test/functional/apps/maps/import_geojson/file_indexing_panel.js b/x-pack/test/functional/apps/maps/import_geojson/file_indexing_panel.js index 0ce9b7022b38..a5b376cbb33a 100644 --- a/x-pack/test/functional/apps/maps/import_geojson/file_indexing_panel.js +++ b/x-pack/test/functional/apps/maps/import_geojson/file_indexing_panel.js @@ -13,6 +13,7 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['maps', 'common']); const log = getService('log'); const security = getService('security'); + const browser = getService('browser'); const IMPORT_FILE_PREVIEW_NAME = 'Import File'; const FILE_LOAD_DIR = 'test_upload_files'; @@ -102,6 +103,9 @@ export default function ({ getService, getPageObjects }) { const pointGeojsonFiles = ['point.json', 'multi_point.json']; pointGeojsonFiles.forEach(async (pointFile) => { it(`should index with type geo_point for file: ${pointFile}`, async () => { + if (!(await browser.checkBrowserPermission('clipboard-read'))) { + return; + } await loadFileAndIndex(pointFile); const indexPatternResults = await PageObjects.maps.getIndexPatternResults(); const coordinatesField = indexPatternResults.fields.find( @@ -120,6 +124,9 @@ export default function ({ getService, getPageObjects }) { ]; nonPointGeojsonFiles.forEach(async (shapeFile) => { it(`should index with type geo_shape for file: ${shapeFile}`, async () => { + if (!(await browser.checkBrowserPermission('clipboard-read'))) { + return; + } await loadFileAndIndex(shapeFile); const indexPatternResults = await PageObjects.maps.getIndexPatternResults(); const coordinatesField = indexPatternResults.fields.find( From cd56eda00d792fb9f577bd1fe0865d3b26528108 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 1 Mar 2021 09:42:55 -0700 Subject: [PATCH 18/26] fix reporting tests --- .../reporting/ecommerce_kibana/data.json | 31 ++ .../reporting/ecommerce_kibana/data.json.gz | Bin 4526 -> 0 bytes .../ecommerce_kibana_spaces/data.json | 362 ++++++++++++++++++ .../ecommerce_kibana_spaces/data.json.gz | Bin 2354 -> 0 bytes 4 files changed, 393 insertions(+) create mode 100644 x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json delete mode 100644 x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json.gz create mode 100644 x-pack/test/functional/es_archives/reporting/ecommerce_kibana_spaces/data.json delete mode 100644 x-pack/test/functional/es_archives/reporting/ecommerce_kibana_spaces/data.json.gz diff --git a/x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json b/x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json new file mode 100644 index 000000000000..a0f384a96e6b --- /dev/null +++ b/x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json @@ -0,0 +1,31 @@ +{ "type": "doc", "value": { "id": "config:7.0.0", "index": ".kibana_1", "source": { "config": { "buildNum": 9007199254740991, "dateFormat:tz": "UTC", "defaultIndex": "5193f870-d861-11e9-a311-0fa548c5f953" }, "migrationVersion": { "config": "7.13.0" }, "references": [ ], "type": "config", "updated_at": "2019-09-16T09:06:51.201Z" } } } + +{ "type": "doc", "value": { "id": "config:8.0.0", "index": ".kibana_1", "source": { "config": { "buildNum": 9007199254740991, "dateFormat:tz": "UTC", "defaultIndex": "5193f870-d861-11e9-a311-0fa548c5f953" }, "migrationVersion": { "config": "7.13.0" }, "references": [ ], "type": "config", "updated_at": "2019-12-11T23:22:12.698Z" } } } + +{ "type": "doc", "value": { "id": "index-pattern:5193f870-d861-11e9-a311-0fa548c5f953", "index": ".kibana_1", "source": { "index-pattern": { "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"category\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"category.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"category\"}}},{\"name\":\"currency\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"customer_birth_date\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"customer_first_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"customer_first_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"customer_first_name\"}}},{\"name\":\"customer_full_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"customer_full_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"customer_full_name\"}}},{\"name\":\"customer_gender\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"customer_id\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"customer_last_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"customer_last_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"customer_last_name\"}}},{\"name\":\"customer_phone\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"day_of_week\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"day_of_week_i\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"email\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.city_name\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.continent_name\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.country_iso_code\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.location\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geoip.region_name\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"manufacturer\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"manufacturer.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"manufacturer\"}}},{\"name\":\"order_date\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"order_id\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products._id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"products._id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"products._id\"}}},{\"name\":\"products.base_price\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.base_unit_price\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.category\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"products.category.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"products.category\"}}},{\"name\":\"products.created_on\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.discount_amount\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.discount_percentage\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.manufacturer\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"products.manufacturer.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"products.manufacturer\"}}},{\"name\":\"products.min_price\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.price\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.product_id\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.product_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"products.product_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"products.product_name\"}}},{\"name\":\"products.quantity\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.sku\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.tax_amount\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.taxful_price\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.taxless_price\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"products.unit_discount_amount\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sku\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"taxful_total_price\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"taxless_total_price\",\"type\":\"number\",\"esTypes\":[\"half_float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"total_quantity\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"total_unique_products\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", "timeFieldName": "order_date", "title": "ecommerce" }, "migrationVersion": { "index-pattern": "7.11.0" }, "references": [ ], "type": "index-pattern", "updated_at": "2019-12-11T23:24:13.381Z" } } } + +{ "type": "doc", "value": { "id": "search:6091ead0-1c6d-11ea-a100-8589bb9d7c6b", "index": ".kibana_1", "source": { "migrationVersion": { "search": "7.9.3" }, "references": [ { "id": "5193f870-d861-11e9-a311-0fa548c5f953", "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern" } ], "search": { "columns": [ "category", "currency", "customer_id", "order_id", "day_of_week_i", "order_date", "products.created_on", "sku" ], "description": "", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "sort": [ [ "order_date", "desc" ] ], "title": "Ecommerce Data", "version": 1 }, "type": "search", "updated_at": "2019-12-11T23:24:28.540Z" } } } + +{ "type": "doc", "value": { "id": "dashboard:constructed-sample-saved-object-id", "index": ".kibana_1", "source": { "dashboard": { "description": "", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" }, "optionsJSON": "{\"hidePanelTitles\":true,\"useMargins\":true}", "panelsJSON": "[{\"version\":\"8.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"1c12c2f2-80c2-4d5c-b722-55b2415006e1\"},\"panelIndex\":\"1c12c2f2-80c2-4d5c-b722-55b2415006e1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15,\"i\":\"1c4b99e1-7785-444f-a1c5-f592893b1a96\"},\"panelIndex\":\"1c4b99e1-7785-444f-a1c5-f592893b1a96\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":48,\"h\":18,\"i\":\"94eab06f-60ac-4a85-b771-3a8ed475c9bb\"},\"panelIndex\":\"94eab06f-60ac-4a85-b771-3a8ed475c9bb\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":0,\"y\":33,\"w\":48,\"h\":8,\"i\":\"52c19b6b-7117-42ac-a74e-c507a1c3ffc0\"},\"panelIndex\":\"52c19b6b-7117-42ac-a74e-c507a1c3ffc0\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":0,\"y\":41,\"w\":11,\"h\":10,\"i\":\"a1e889dc-b80e-4937-a576-979f34d1859b\"},\"panelIndex\":\"a1e889dc-b80e-4937-a576-979f34d1859b\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":11,\"y\":41,\"w\":5,\"h\":10,\"i\":\"4930b035-d756-4cc5-9a18-1af9e67d6f31\"},\"panelIndex\":\"4930b035-d756-4cc5-9a18-1af9e67d6f31\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"}]", "refreshInterval": { "pause": true, "value": 0 }, "timeFrom": "2019-06-26T06:20:28.066Z", "timeRestore": true, "timeTo": "2019-06-26T07:27:58.573Z", "title": "Ecom Dashboard Hidden Panel Titles", "version": 1 }, "migrationVersion": { "dashboard": "7.11.0" }, "references": [ { "id": "0a464230-79f0-11ea-ae7f-13c5d6e410a0", "name": "panel_0", "type": "visualization" }, { "id": "200609c0-79f0-11ea-ae7f-13c5d6e410a0", "name": "panel_1", "type": "visualization" }, { "id": "6091ead0-1c6d-11ea-a100-8589bb9d7c6b", "name": "panel_2", "type": "search" }, { "id": "4a36acd0-7ac3-11ea-b69c-cf0d7935cd67", "name": "panel_3", "type": "visualization" }, { "id": "ef8757d0-7ac2-11ea-b69c-cf0d7935cd67", "name": "panel_4", "type": "visualization" }, { "id": "132ab9c0-7ac3-11ea-b69c-cf0d7935cd67", "name": "panel_5", "type": "visualization" } ], "type": "dashboard", "updated_at": "2020-04-10T00:37:48.462Z" } } } + +{ "type": "doc", "value": { "id": "visualization:0a464230-79f0-11ea-ae7f-13c5d6e410a0", "index": ".kibana_1", "source": { "migrationVersion": { "visualization": "7.12.0" }, "references": [ { "id": "5193f870-d861-11e9-a311-0fa548c5f953", "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern" } ], "type": "visualization", "updated_at": "2020-04-08T23:24:05.971Z", "visualization": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "title": "e-commerce area chart", "uiStateJSON": "{}", "version": 1, "visState": "{\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"order_date\",\"timeRange\":{\"from\":\"2019-06-26T06:20:28.066Z\",\"to\":\"2019-06-26T07:27:58.573Z\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{},\"isVislibVis\":true,\"detailedTooltip\":true,\"fittingFunction\":\"zero\"},\"title\":\"e-commerce area chart\"}" } } } } + +{ "type": "doc", "value": { "id": "visualization:200609c0-79f0-11ea-ae7f-13c5d6e410a0", "index": ".kibana_1", "source": { "migrationVersion": { "visualization": "7.12.0" }, "references": [ { "id": "5193f870-d861-11e9-a311-0fa548c5f953", "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern" } ], "type": "visualization", "updated_at": "2020-04-08T23:24:42.460Z", "visualization": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "title": "e-commerce pie chart", "uiStateJSON": "{}", "version": 1, "visState": "{\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"order_date\",\"timeRange\":{\"from\":\"2019-06-26T06:20:28.066Z\",\"to\":\"2019-06-26T07:27:58.573Z\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"title\":\"e-commerce pie chart\"}" } } } } + +{ "type": "doc", "value": { "id": "visualization:ef8757d0-7ac2-11ea-b69c-cf0d7935cd67", "index": ".kibana_1", "source": { "migrationVersion": { "visualization": "7.12.0" }, "references": [ { "id": "5193f870-d861-11e9-a311-0fa548c5f953", "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern" } ], "type": "visualization", "updated_at": "2020-04-10T00:33:44.909Z", "visualization": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\",\"filter\":[]}" }, "title": "게이지", "uiStateJSON": "{}", "version": 1, "visState": "{\"type\":\"gauge\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"params\":{\"type\":\"gauge\",\"addTooltip\":true,\"addLegend\":true,\"isDisplayWarning\":false,\"gauge\":{\"alignment\":\"automatic\",\"extendRange\":true,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Labels\",\"colorsRange\":[{\"from\":0,\"to\":50},{\"from\":50,\"to\":75},{\"from\":75,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"rgba(105,112,125,0.2)\"},\"type\":\"meter\",\"style\":{\"bgWidth\":0.9,\"width\":0.9,\"mask\":false,\"bgMask\":false,\"maskBars\":50,\"bgFill\":\"rgba(105,112,125,0.2)\",\"bgColor\":true,\"subText\":\"\",\"fontSize\":60}}},\"title\":\"게이지\"}" } } } } + +{ "type": "doc", "value": { "id": "visualization:132ab9c0-7ac3-11ea-b69c-cf0d7935cd67", "index": ".kibana_1", "source": { "migrationVersion": { "visualization": "7.12.0" }, "references": [ { "id": "5193f870-d861-11e9-a311-0fa548c5f953", "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern" } ], "type": "visualization", "updated_at": "2020-04-10T00:34:44.700Z", "visualization": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\",\"filter\":[]}" }, "title": "Українська", "uiStateJSON": "{}", "version": 1, "visState": "{\"type\":\"metric\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"title\":\"Українська\"}" } } } } + +{ "type": "doc", "value": { "id": "visualization:4a36acd0-7ac3-11ea-b69c-cf0d7935cd67", "index": ".kibana_1", "source": { "migrationVersion": { "visualization": "7.12.0" }, "references": [ ], "type": "visualization", "updated_at": "2020-04-10T00:36:17.053Z", "visualization": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" }, "title": "Tiểu thuyết", "uiStateJSON": "{}", "version": 1, "visState": "{\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":12,\"openLinksInNewTab\":false,\"markdown\":\"Tiểu thuyết là một thể loại văn xuôi có hư cấu, thông qua nhân vật, hoàn cảnh, sự việc để phản ánh bức tranh xã hội rộng lớn và những vấn đề của cuộc sống con người, biểu hiện tính chất tường thuật, tính chất kể chuyện bằng ngôn ngữ văn xuôi theo những chủ đề xác định.\\n\\nTrong một cách hiểu khác, nhận định của Belinski: \\\"tiểu thuyết là sử thi của đời tư\\\" chỉ ra khái quát nhất về một dạng thức tự sự, trong đó sự trần thuật tập trung vào số phận của một cá nhân trong quá trình hình thành và phát triển của nó. Sự trần thuật ở đây được khai triển trong không gian và thời gian nghệ thuật đến mức đủ để truyền đạt cơ cấu của nhân cách[1].\\n\\n\\n[1]^ Mục từ Tiểu thuyết trong cuốn 150 thuật ngữ văn học, Lại Nguyên Ân biên soạn, Nhà xuất bản Đại học Quốc gia Hà Nội, in lần thứ 2 có sửa đổi bổ sung. H. 2003. Trang 326.\"},\"title\":\"Tiểu thuyết\"}" } } } } + +{ "type": "doc", "value": { "id": "space:default", "index": ".kibana_1", "source": { "space": { "_reserved": true, "description": "This is the default space", "disabledFeatures": [ ], "name": "Default Space" }, "type": "space", "updated_at": "2021-01-07T00:17:12.785Z" } } } + +{ "type": "doc", "value": { "id": "ui-counter:visualize:06012021:click:tagcloud", "index": ".kibana_1", "source": { "type": "ui-counter", "ui-counter": { "count": 1 }, "updated_at": "2021-01-07T00:18:52.592Z" } } } + +{ "type": "doc", "value": { "id": "ui-counter:data_plugin:06012021:click:discover:query_submitted", "index": ".kibana_1", "source": { "type": "ui-counter", "ui-counter": { "count": 1 }, "updated_at": "2021-01-07T00:18:52.592Z" } } } + +{ "type": "doc", "value": { "id": "dashboard:6c263e00-1c6d-11ea-a100-8589bb9d7c6b", "index": ".kibana_1", "source": { "dashboard": { "description": "", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" }, "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "panelsJSON": "[{\"version\":\"8.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"1c12c2f2-80c2-4d5c-b722-55b2415006e1\"},\"panelIndex\":\"1c12c2f2-80c2-4d5c-b722-55b2415006e1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15,\"i\":\"1c4b99e1-7785-444f-a1c5-f592893b1a96\"},\"panelIndex\":\"1c4b99e1-7785-444f-a1c5-f592893b1a96\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":0,\"y\":35,\"w\":48,\"h\":18,\"i\":\"94eab06f-60ac-4a85-b771-3a8ed475c9bb\"},\"panelIndex\":\"94eab06f-60ac-4a85-b771-3a8ed475c9bb\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":48,\"h\":8,\"i\":\"52c19b6b-7117-42ac-a74e-c507a1c3ffc0\"},\"panelIndex\":\"52c19b6b-7117-42ac-a74e-c507a1c3ffc0\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":0,\"y\":23,\"w\":16,\"h\":12,\"i\":\"a1e889dc-b80e-4937-a576-979f34d1859b\"},\"panelIndex\":\"a1e889dc-b80e-4937-a576-979f34d1859b\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":null},\"panelRefName\":\"panel_4\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":16,\"y\":23,\"w\":12,\"h\":12,\"i\":\"4930b035-d756-4cc5-9a18-1af9e67d6f31\"},\"panelIndex\":\"4930b035-d756-4cc5-9a18-1af9e67d6f31\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"8.0.0\",\"gridData\":{\"x\":28,\"y\":23,\"w\":20,\"h\":12,\"i\":\"55112375-d6f0-44f7-a8fb-867c8f7d464d\"},\"panelIndex\":\"55112375-d6f0-44f7-a8fb-867c8f7d464d\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"}]", "refreshInterval": { "pause": true, "value": 0 }, "timeFrom": "2019-03-23T03:06:17.785Z", "timeRestore": true, "timeTo": "2019-10-04T02:33:16.708Z", "title": "Ecom Dashboard", "version": 1 }, "migrationVersion": { "dashboard": "7.11.0" }, "references": [ { "id": "0a464230-79f0-11ea-ae7f-13c5d6e410a0", "name": "panel_0", "type": "visualization" }, { "id": "200609c0-79f0-11ea-ae7f-13c5d6e410a0", "name": "panel_1", "type": "visualization" }, { "id": "6091ead0-1c6d-11ea-a100-8589bb9d7c6b", "name": "panel_2", "type": "search" }, { "id": "4a36acd0-7ac3-11ea-b69c-cf0d7935cd67", "name": "panel_3", "type": "visualization" }, { "id": "ef8757d0-7ac2-11ea-b69c-cf0d7935cd67", "name": "panel_4", "type": "visualization" }, { "id": "132ab9c0-7ac3-11ea-b69c-cf0d7935cd67", "name": "panel_5", "type": "visualization" }, { "id": "1bba55f0-507e-11eb-9c0d-97106882b997", "name": "panel_6", "type": "visualization" } ], "type": "dashboard", "updated_at": "2021-01-07T00:22:16.102Z" } } } + +{ "type": "doc", "value": { "id": "visualization:1bba55f0-507e-11eb-9c0d-97106882b997", "index": ".kibana_1", "source": { "migrationVersion": { "visualization": "7.12.0" }, "references": [ { "id": "6091ead0-1c6d-11ea-a100-8589bb9d7c6b", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-01-07T00:23:04.624Z", "visualization": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" }, "savedSearchRefName": "search_0", "title": "Tag Cloud of Names", "uiStateJSON": "{}", "version": 1, "visState": "{\"title\":\"Tag Cloud of Names\",\"type\":\"tagcloud\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"customer_first_name.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"scale\":\"linear\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":true}}" } } } } + +{ "type": "doc", "value": { "id": "ui-counter:DashboardPanelVersionInUrl:06012021:loaded:8.0.0", "index": ".kibana_1", "source": { "type": "ui-counter", "ui-counter": { "count": 85 }, "updated_at": "2021-01-07T00:23:25.741Z" } } } diff --git a/x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json.gz b/x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json.gz deleted file mode 100644 index 8b2b8f9c114de2968d1ebb6395c1af082486c6a0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4526 zcmV;f5mD|RiwFqyRQ6v217u-zVJ>QOZ*BnXUCnRg#+AS4uOK>?U7*=0emDDOk3DNI zMzUjdCV)q?jUrjyUE6H3%@2KeARyixf<+QweFzr8!dOoXW02h)th4b+!XgJ5K9Bzd zT^IQy_EiEGt%lwEiZIiZK<_~{ghol zlUmec?QR$h>8NLQPzSLNH1Nm)yV03Y2M2hdu+}(CBR3l-Mu^W&N_}rTox+ukkfqqR zYUq}ZY+Hd*9!|)^Fq-0|mwd`-{|4V-m&iqlTzub!=dRca6x9L?m!Y3^$%gPup`@Wc{Hf zoiU8w04J7A^04-5uJfZ;Btuln6M^-BX^7)pOH(Vl(pf}C?X@A zR9hp0y@ydaeJ6AuGr`0;m7{hQA!DW~NYVFJaK>&Jkt)t0ojN4qZR^0v8*6y-=Ei`y zMN|=$8>^zrZ#=ynumWKtC5s-np<@@;|0s+On*mM8K~lx3TpH(^;>>||CggBGj2f=- zqVr!}w%M&Xb@~DnfS-bwr{d!bgELNQB0oB+@t8);WglLFH|4LzTODbfgi{g?92zC# z0gED4&_zoHvS_^#U97-6q*0sm-B%vaQI|B!vc}jS z4u*b+Z^zqLai;-IHWB<)&tPwUNxVLXu}hS&a&fpY6?|t?G?R>z1N^jaK*hiQc>$4K^`=Y$Rm2sagfmD@CrwUH*IF|B)$7b76sY_A8cfH0l0^$zlpvckozQ#HxWAD zpKW4NdbKrqLur3z(n*-$O)~7kkd;VY_WS|nlL!1y@KnqwCAj| z(XjN!bTszifAW^^&)318(9^sDoE{zNQ8-}Cm zih+Saf;7-x3k{NHE6W+orG&XWTwJDyAr}4c;(748R(E zb3QL-Tz71nD6(aB4O!RqA^3FHkcWn?c5Tg3ux+kjyve|8VO&|0@m%4YO~$z1%^B|& zjN3ZF4l;+biLfi{7?^M@OOZ9)C7x~>E`+qDj5iy2EsU#|WL(qgjL-QtR9CSb(~&Jj zv1DBZ9!8le(w`+T_KD&s>x~*9< zHY`)NEqkcxp3*h!fDRWE>D92Wf`vS%xX=E-X?TD_vQ^Lz|eE zXAZUHzOC87Yhl~~06e9JbT1;hQ}yP6>Dn6@jsL*!YuX1Jbd zEt1t5f+n!PFf2h`ZA@KnNL|rX>~MjvgWWifdSzcYx348xtcql$%L?iv)YB}86CK@D z*UDlmbN8Cs=4H2rRZhi%Q7>9Bx4K1KuA`NzK;3LZfsBrADctSUnO>gSUn!rz#l747 zbVM$;Wf=Cjk~_w*t7DR)I|+!YKz>wnu0lk-PsUksO2i0pj__#24~XC<07N_U2J0?>s>KREAQJ1&QaD=CMK%0|QFCfd5J{VJoKVaX}I{lc8rX-to zKEbYd6?{B)&w0coE8vt7gg>I$ZtSSKf%bUO7+08QbaZjW18fdnG+^Sn5l9c?w2H# z9P&fNXGgiXYN&mHh9eJF!#Vi!MD4PtIpWrj_qi>;O_F^=f)7~@f9bq~ ziRWk3OIX^jX3~8?Be!xW!WY&o^aZ;WneOH4T~s#b8GBwo4E==8N~hhP_lU4J3rjHg z^@nAJN9-K1a5r)8{~1BJ-onwOs*f>C-9z7F+*Z_?_s9%O>`wMGo|TdfftFL>&wdMn zA4a_C$M0IEsXwS23%jE6V;cL^c}z>P@*w(9@WKmR4{4InVDu2orr_~Y5`}_tZallo zBa#SQm(Gxy=GXTb(nT4Pu0pbk-p~wbM#;4#NI>U4KY9c5qe9Qu5CmxaP8g&~srO>w zu6X)#kYg|S`ecvzEAyF^KDe;?E4tvO`}unpeExZX*3 z&#(XT#kYTX@$)PC-4RYlH{p3#I)oB{W-b9%J&k8RKKvPu0$zGiEb@#NqYox93Z{8E zDf4Voun2)mxUAe8XJqGZA8d&PyF#>|!+k3fGQ=SO~pI(d5-vua1BC z`Kxb!_}5qe`@?5H{QA}Lt8cCtSW0o=W+Tf*VWc=6Sb{YF$lpd-6%Zid=FWD(bDihH z1?8#_-_C!3%*7rYJaXofS2*9J)$Y3D9SeW_UlspeZ(Pr*EiNTp<)9-bZl`KtT;DUK#yxF1v;LwyOCOiN)Wkl~C3 zk7zJ~&FwbX?_;MDvU7q!y@pltPrsC=XD7c)pb73set35LoJxD&{UVSK($jCL9@g1`Xt3tFh2c#AnlzUzf9WFI6VC_V4c1S#%(D+JNcuu2Xp<> zmA?C&5uT0NE$Q@mFqWLNlRvmp5@Gm!aQc-5GyjfC5qt;A{@KaD0ad^iK>L3~3(W9! zz{z|D6P&!jlAA(bm(B7y+;_u33P#_5b9V9tZA*^eaLgtL2Xp!oCUIf%uYpD02Sxz< z0(Ir~gyD3VWY|dO?D&t+H5dWE@ExF(rIL(ESOA3aCoeL92dB>k%fAZ7o!#94zWPxJ zorIKKxIO0J0OMo~S8Yb-_+^j}%2<4x_%w(ov?uNEGQY8?HsJmxw9sq-8~z1lJhA~C z(?3fQ=95s+*Xi?w&jgI_F`xpE2kPbm5h`J7W4Z&$aM0g<4%+9+Oro>n7eOvL=z08d z#%eN{eons(xh}YpUj`YVg5k_i1Vsh}KA!#;a60DS!J@u|A57!3F+&M)0JzP`1gC%M zNIOd)oSpnT!}t3`_|6pjl?%LLS_~KSO@zIUC>BP=I0Z5B>tF;IzbpuX1dsn5Fx-C- zDtwXIF0ctU@flaqF#vSGe=e3o4w3W6Rr^GFnrR)r;Nl;ot+SJ_xn%!SsxLl)kQwvm zfutCyKvFU{K=>Q5kw<)4ZjaK#)Bk`~^iN=)lzocXdJWprHV6?+ldIR^EA6{~;Y0X% z=_hOsmvJq<2OYQBf@@1OfYl~M3Oq|HU+c`Qgf;z^vX%NR1Sk%h%&-JN0=|KwS`eE9z_Bfw$PFX~(;wmeWhl`>h>Ri6MJwxpnwt88<*rU0@ z%uh!&sB^|&k%PF|<57A5Db|$24rZe`omY>Wu4-xoUBl}buSxxQX~S?|Kz?69e)R%! z?M*EpU#y&eGYZJnyl`GIbA_vSuYjCAUgPx+LF)V0Sxs&(h(TQx!={vzujT2wb?$D9DrsZ~rmZzJ#x7_j@4ZIf4&Fd_r*JM@eBaQt9AjJ~7 zX2U{yg`L6lk;)zxDrUz*-SZ0R?~CT|i{@{wXueF{aj;>4CLtmbM%|HNE)N{Fg3NAL zg?wcRb@Q4^@=IF-_9vpuj)JaIlHZh>8+|^tu1O2~t**zyrq?_cF3T5un%*&0{mvf? z^Y?*wIQ`7oFNmc$IL0IC0o%As;ZR~d<4c|$&re_$C|2u)^OJ%rKXocSUt0bQYMn|l zkl-{vr&8>qKmWoj$=*IJvA|nIl^WQEcutlbm$Nh>OvW%Ebtfb#?LtefkBS=M4_Kpk zKtxmG=3-Ezz7m;a@C_TX-&)mjP62h3j<@jHXXAK*7V(bAT>C>pv(pS$T1C zj*4%s*F2Of9n=)zpYP%`-|~L&)5xEn+xQ{&h_}vP5nS(-rfXbyOv8#!jgF3MnCs{}#^y%%Jj$ zV_e!!f~OjtQFwL*y;0m(xoD@Gjv>?g!I^p|#LZPt&*N*hu&sUKX5v?(EGq(dU$+N< z;gDqRuuSBRU8*Gy!@y<+=}D7VnrlM0#n#)lk|sflY@7HekL!AUcsisaim& zn1){ZDFu@;KJvwKS@B6++3^k3<6>YFIWe_6^3~%u?pa|}^fcgd3<#w<0DRuu4Z{^S zUvVhl*u+()2f74IIiCo8i#I!MXuVZ_v604@%h&XQMyGw+2%0LDU%Gif!(w5AiU%1;O&fy;L5K@$2`(AW%F;V2Hip;fc`=QYI|?e~X{XZOoH@4xL_a?@R z%2KDj&+s|R9;k#pYeefXN~Z5}(z@2U^r$yQ@jB zG&_D6Zqy7kgn?AEoW<6(vq$(rK!xX)Krz{CR!E&mHVZ| zdQqi+zn(yK|Cgv^k9B$efAWXbKj{p2hss%?os%=U9^Fy`YmEF6FKs#|)|aaSj11m) zQ4(Vs8}%8;66;SN+FmAPis(fP+FQ}~6@GPjq9EF40tdqKbT+ za}TDQ@&XKi%mI@7!N1JMp6-5lrAhf~>df8ni;Q0g||U-)e|*NPbce27lO?x5-XB*WL}# zm-dQke!1lpv_DtYX;=Kj(1x+S)qxds@qw)HJIXY_K)<8&*(a?tI^?}lb0R)yfhXeY zoN5a?X^#=|&>gX4jq;e^cWvUG}l#O!T4m;tyOSUjN!Y(po$$I`K0U^~+aW#zvVx&4Y9&JlgCe#k94x|yx1c37ajF0iW~@bc#oS!g z%GICk5mc%T9bIdUfI$tb_8X5wHrcQNPG#oeyRYo@7xgyFiXGa{^Sqh)_#>4riJNr@ zHxE$B;);!d^%Ux;E2>7BykwF*DAuUk^i^DxPzi}wsm6VOAIgZ+$bz|r|D5praN5q8 z5g(UC81cirPn}^H^OmYAt7)?xS$B1*aHD%TTMbM^&dv=Q2;H%=s1H$4hdZa(X*T8F z)$#GVY%7y9OIRBN?NrnT$b5MOhzU6bT|SWNMq@i!-_&$|S=MDNd{PcctSPQUXT(V? zCN!7?f)ATfJM%N&56)I88A=(z^10vpIWCqv4+wnNt6}89qxZ}Ud9ta1Xg2pnH}2>i zSsJmEHUJl^oNxuy=xxXI0#`Fr%+Gpy>r~nzPB}5~lyZd7lLviDV#|~>x~s3tIwFOT z>wTsx3U@@UN>_SW`sh!C9%VL1L^i;iPx8DruJ!vvSh7RqGr8d?rDuk4iud}CNi{F$ zyMbcehwRa@G~O0pIRynkjAZ@#Y5Vhk1OLCS&^mDzPP&DB?f+E&#dqV@Y=81U#(!gq z?K`~dnE%-rQY-oY)Jv8rjMjSO+w-`aDV#+ioJJl6OZZ_V4Yc$^^ z_D=u?zBk`M4%Us$W~+xD|3=PahuX=>1~Yxji9@sPSF2J)U3`VUIe|z^Ryy{j Date: Mon, 1 Mar 2021 10:38:49 -0700 Subject: [PATCH 19/26] review feedback --- .../importer/geojson_importer/geojson_importer.ts | 11 +++++------ .../file_upload/public/importer/validate_file.ts | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts index 95c81bfac357..dd8f0102aa18 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts @@ -24,6 +24,7 @@ import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; import { geoJsonCleanAndValidate } from './geojson_clean_and_validate'; import { ImportFailure, ImportResponse, MB } from '../../../common'; +const IMPORT_CHUNK_SIZE_MB = 10 * MB; export const GEOJSON_FILE_TYPES = ['.json', '.geojson']; export class GeoJsonImporter extends Importer { @@ -87,16 +88,12 @@ export class GeoJsonImporter extends Importer { }; } - const ingestPipeline = { - id: pipelineId, - }; - let success = true; const failures: ImportFailure[] = [...this._invalidFeatures]; let error; while (this._features.length > 0 || (this._hasNext && this._isActive)) { - await this._readUntil(undefined, 10 * MB); + await this._readUntil(undefined, IMPORT_CHUNK_SIZE_MB); if (!this._isActive) { return { success: false, @@ -127,7 +124,9 @@ export class GeoJsonImporter extends Importer { data, settings: {}, mappings: {}, - ingestPipeline, + ingestPipeline: { + id: pipelineId, + }, }); if (retries < IMPORT_RETRIES) { diff --git a/x-pack/plugins/file_upload/public/importer/validate_file.ts b/x-pack/plugins/file_upload/public/importer/validate_file.ts index 98456d98d724..4c7fe704d8af 100644 --- a/x-pack/plugins/file_upload/public/importer/validate_file.ts +++ b/x-pack/plugins/file_upload/public/importer/validate_file.ts @@ -44,8 +44,8 @@ export function validateFile(file: File, types: string[]) { } function bytesToSize(bytes: number) { - const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; if (bytes === 0) return 'n/a'; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; const i = Math.round(Math.floor(Math.log(bytes) / Math.log(1024))); if (i === 0) return `${bytes} ${sizes[i]})`; return `${(bytes / 1024 ** i).toFixed(1)} ${sizes[i]}`; From 56957f426c910c1bedc3c31699fbf81bd0f22503 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 1 Mar 2021 13:19:44 -0700 Subject: [PATCH 20/26] update GeoJsonFileSource to support showing results trimmed icon and tooltip --- .../public/components/geojson_file_picker.tsx | 2 ++ .../public/components/json_upload_and_parse.js | 5 +++-- .../file_upload/public/lazy_load_bundle/index.ts | 2 +- .../descriptor_types/source_descriptor_types.ts | 2 ++ .../classes/layers/file_upload_wizard/wizard.tsx | 14 +++++++++++++- .../geojson_file_source/geojson_file_source.ts | 10 ++++++++++ 6 files changed, 31 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx index 5776c688d569..54ca0de91b5f 100644 --- a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx +++ b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx @@ -20,11 +20,13 @@ interface Props { geoFieldTypes, importer, indexName, + previewCoverage, }: { features: Feature[]; indexName: string; importer: GeoJsonImporter; geoFieldTypes: string[]; + previewCoverage: number; }) => void; onClear: () => void; } diff --git a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js index d6d1ce615bf0..018d09a1b2f3 100644 --- a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js +++ b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js @@ -201,7 +201,7 @@ export class JsonUploadAndParse extends Component { }); }; - _onFileSelect = ({ features, geoFieldTypes, importer, indexName }) => { + _onFileSelect = ({ features, geoFieldTypes, importer, indexName, previewCoverage }) => { this._geojsonImporter = importer; const newState = { @@ -225,7 +225,8 @@ export class JsonUploadAndParse extends Component { type: 'FeatureCollection', features, }, - indexName + indexName, + previewCoverage ); }; diff --git a/x-pack/plugins/file_upload/public/lazy_load_bundle/index.ts b/x-pack/plugins/file_upload/public/lazy_load_bundle/index.ts index 9cfc0896f5c2..a61faa5bef0c 100644 --- a/x-pack/plugins/file_upload/public/lazy_load_bundle/index.ts +++ b/x-pack/plugins/file_upload/public/lazy_load_bundle/index.ts @@ -12,7 +12,7 @@ import { IImporter, ImportFactoryOptions, ImportResults } from '../importer'; export interface FileUploadComponentProps { isIndexingTriggered: boolean; - onFileUpload: (geojsonFile: FeatureCollection, name: string) => void; + onFileUpload: (geojsonFile: FeatureCollection, name: string, previewCoverage: number) => void; onFileRemove: () => void; onIndexReady: (indexReady: boolean) => void; onIndexingComplete: (results: { diff --git a/x-pack/plugins/maps/common/descriptor_types/source_descriptor_types.ts b/x-pack/plugins/maps/common/descriptor_types/source_descriptor_types.ts index c825f148cd0c..7b757aa9cf10 100644 --- a/x-pack/plugins/maps/common/descriptor_types/source_descriptor_types.ts +++ b/x-pack/plugins/maps/common/descriptor_types/source_descriptor_types.ts @@ -174,6 +174,8 @@ export type InlineFieldDescriptor = { export type GeojsonFileSourceDescriptor = { __fields?: InlineFieldDescriptor[]; __featureCollection: FeatureCollection; + areResultsTrimmed: boolean; + tooltipContent: string | null; name: string; type: string; }; diff --git a/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx b/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx index 138ed7a8cd0b..82a5298d5b42 100644 --- a/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx +++ b/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx @@ -5,6 +5,7 @@ * 2.0. */ +import { i18n } from '@kbn/i18n'; import React, { Component } from 'react'; import { FeatureCollection } from 'geojson'; import { EuiPanel } from '@elastic/eui'; @@ -70,7 +71,7 @@ export class ClientFileCreateSourceEditor extends Component { + _onFileUpload = (geojsonFile: FeatureCollection, name: string, previewCoverage: number) => { if (!this._isMounted) { return; } @@ -80,8 +81,19 @@ export class ClientFileCreateSourceEditor extends Component Date: Tue, 2 Mar 2021 08:26:59 -0700 Subject: [PATCH 21/26] add fileUpload to useMlKibana context and replace dependencyCache with useMlKibana --- x-pack/plugins/ml/public/application/app.tsx | 1 + .../public/application/contexts/kibana/kibana_context.ts | 2 ++ .../datavisualizer/datavisualizer_selector.tsx | 5 +++-- .../components/about_panel/welcome_content.tsx | 9 +++++++-- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ml/public/application/app.tsx b/x-pack/plugins/ml/public/application/app.tsx index 107bbda23ecb..5f72d49e4672 100644 --- a/x-pack/plugins/ml/public/application/app.tsx +++ b/x-pack/plugins/ml/public/application/app.tsx @@ -82,6 +82,7 @@ const App: FC = ({ coreStart, deps, appMountParams }) => { embeddable: deps.embeddable, maps: deps.maps, triggersActionsUi: deps.triggersActionsUi, + fileUpload: deps.fileUpload, ...coreStart, }; diff --git a/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts b/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts index 99d4b77547d9..3c86bb520600 100644 --- a/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts +++ b/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts @@ -18,6 +18,7 @@ import { MlServicesContext } from '../../app'; import { IStorageWrapper } from '../../../../../../../src/plugins/kibana_utils/public'; import type { EmbeddableStart } from '../../../../../../../src/plugins/embeddable/public'; import type { MapsStartApi } from '../../../../../maps/public'; +import type { FileUploadPluginStart } from '../../../../../file_upload/public'; import type { LensPublicStart } from '../../../../../lens/public'; import { TriggersAndActionsUIPublicPluginStart } from '../../../../../triggers_actions_ui/public'; @@ -30,6 +31,7 @@ interface StartPlugins { maps?: MapsStartApi; lens?: LensPublicStart; triggersActionsUi?: TriggersAndActionsUIPublicPluginStart; + fileUpload?: FileUploadPluginStart; } export type StartServices = CoreStart & StartPlugins & { diff --git a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx index 3f620ae16d57..03bdb23d69a4 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx @@ -27,7 +27,6 @@ import { isFullLicense } from '../license'; import { useTimefilter, useMlKibana, useNavigateToPath } from '../contexts/kibana'; import { NavigationMenu } from '../components/navigation_menu'; import { HelpMenu } from '../components/help_menu'; -import { getFileUpload } from '../util/dependency_cache'; function startTrialDescription() { return ( @@ -57,8 +56,10 @@ export const DatavisualizerSelector: FC = () => { licenseManagement, http: { basePath }, docLinks, + fileUpload, }, } = useMlKibana(); + const helpLink = docLinks.links.ml.guide; const navigateToPath = useNavigateToPath(); @@ -67,7 +68,7 @@ export const DatavisualizerSelector: FC = () => { licenseManagement.enabled === true && isFullLicense() === false; - const maxFileSize = getFileUpload().getMaxBytesFormatted(); + const maxFileSize = fileUpload.getMaxBytesFormatted(); return ( diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx index 6748f2e21e3d..558d8fe82c13 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx @@ -20,7 +20,8 @@ import { } from '@elastic/eui'; import { ExperimentalBadge } from '../experimental_badge'; -import { getFileUpload } from '../../../../util/dependency_cache'; + +import { useMlKibana } from '../../../../contexts/kibana'; export const WelcomeContent: FC = () => { const toolTipContent = i18n.translate( @@ -30,7 +31,11 @@ export const WelcomeContent: FC = () => { } ); - const maxFileSize = getFileUpload().getMaxBytesFormatted(); + const { + services: { fileUpload }, + } = useMlKibana(); + + const maxFileSize = fileUpload.getMaxBytesFormatted(); return ( From 7f4ce1c8fee5bb698305a20fff4324f64e33c16f Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 2 Mar 2021 09:27:49 -0700 Subject: [PATCH 22/26] tslint --- .../application/datavisualizer/datavisualizer_selector.tsx | 5 +++++ .../file_based/components/about_panel/welcome_content.tsx | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx index 03bdb23d69a4..fc03ff475e05 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx @@ -68,6 +68,11 @@ export const DatavisualizerSelector: FC = () => { licenseManagement.enabled === true && isFullLicense() === false; + if (fileUpload === undefined) { + // eslint-disable-next-line no-console + console.error('File upload plugin not available'); + return null; + } const maxFileSize = fileUpload.getMaxBytesFormatted(); return ( diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx index 558d8fe82c13..2c441e42dea2 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx @@ -35,6 +35,11 @@ export const WelcomeContent: FC = () => { services: { fileUpload }, } = useMlKibana(); + if (fileUpload === undefined) { + // eslint-disable-next-line no-console + console.error('File upload plugin not available'); + return null; + } const maxFileSize = fileUpload.getMaxBytesFormatted(); return ( From 812b511f5bd8194c1c31aca9803bc95a6c9b6bd0 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 2 Mar 2021 13:14:19 -0700 Subject: [PATCH 23/26] review feedback --- .../file_upload/public/components/import_complete_view.tsx | 4 ++-- .../public/importer/geojson_importer/geojson_importer.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/file_upload/public/components/import_complete_view.tsx b/x-pack/plugins/file_upload/public/components/import_complete_view.tsx index c393e2d4bb0a..c6209ae765c7 100644 --- a/x-pack/plugins/file_upload/public/components/import_complete_view.tsx +++ b/x-pack/plugins/file_upload/public/components/import_complete_view.tsx @@ -103,7 +103,7 @@ export class ImportCompleteView extends Component { }, }); - const failuredFeaturesMsg = this.props.importResp.failures.length + const failedFeaturesMsg = this.props.importResp.failures.length ? i18n.translate('xpack.fileUpload.failedFeaturesMsg', { defaultMessage: 'Unable to index {numFailures} features.', values: { @@ -112,7 +112,7 @@ export class ImportCompleteView extends Component { }) : ''; - return `${successMsg} ${failuredFeaturesMsg}`; + return `${successMsg} ${failedFeaturesMsg}`; } render() { diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts index dd8f0102aa18..2cb2a10b9560 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts @@ -92,7 +92,7 @@ export class GeoJsonImporter extends Importer { const failures: ImportFailure[] = [...this._invalidFeatures]; let error; - while (this._features.length > 0 || (this._hasNext && this._isActive)) { + while ((this._features.length > 0 || this._hasNext) && this._isActive) { await this._readUntil(undefined, IMPORT_CHUNK_SIZE_MB); if (!this._isActive) { return { From e80285a110a346e7821fe4f2d1e4e9a52ea5d473 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 2 Mar 2021 13:15:40 -0700 Subject: [PATCH 24/26] lower case file name --- .../file_upload/public/components/geojson_file_picker.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx index 54ca0de91b5f..30e2819c70eb 100644 --- a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx +++ b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx @@ -114,7 +114,7 @@ export class GeoJsonFilePicker extends Component { this.props.onSelect({ ...preview, importer, - indexName: file.name.split('.')[0], + indexName: file.name.split('.')[0].toLowerCase(), }); } } From eedb74ef33ba96aafe4291e862f482c2ba749103 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 2 Mar 2021 13:58:53 -0700 Subject: [PATCH 25/26] default to selecting geo_shape when file contains both points and shapes --- .../public/components/geojson_file_picker.tsx | 17 +++++--------- .../public/components/index_settings.js | 1 + .../components/json_upload_and_parse.js | 13 ++++++++--- .../geojson_importer/geojson_importer.test.js | 18 ++++++++++----- .../geojson_importer/geojson_importer.ts | 22 ++++++++++++------- .../public/importer/geojson_importer/index.ts | 2 +- 6 files changed, 43 insertions(+), 30 deletions(-) diff --git a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx index 30e2819c70eb..724a23a43202 100644 --- a/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx +++ b/x-pack/plugins/file_upload/public/components/geojson_file_picker.tsx @@ -6,27 +6,24 @@ */ import React, { Component } from 'react'; -import { Feature } from 'geojson'; import { EuiFilePicker, EuiFormRow } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { MB } from '../../common'; import { getMaxBytesFormatted } from '../get_max_bytes'; import { validateFile } from '../importer'; -import { GeoJsonImporter, GEOJSON_FILE_TYPES } from '../importer/geojson_importer'; +import { GeoJsonImporter, GeoJsonPreview, GEOJSON_FILE_TYPES } from '../importer/geojson_importer'; interface Props { onSelect: ({ features, - geoFieldTypes, + hasPoints, + hasShapes, importer, indexName, previewCoverage, - }: { - features: Feature[]; + }: GeoJsonPreview & { indexName: string; importer: GeoJsonImporter; - geoFieldTypes: string[]; - previewCoverage: number; }) => void; onClear: () => void; } @@ -73,11 +70,7 @@ export class GeoJsonFilePicker extends Component { let importer: GeoJsonImporter | null = null; let previewError: string | null = null; - let preview: { - features: Feature[]; - geoFieldTypes: string[]; - previewCoverage: number; - } | null = null; + let preview: GeoJsonPreview | null = null; try { validateFile(file, GEOJSON_FILE_TYPES); importer = new GeoJsonImporter(file); diff --git a/x-pack/plugins/file_upload/public/components/index_settings.js b/x-pack/plugins/file_upload/public/components/index_settings.js index 3cad31cdbe3d..7d7ab3516e65 100644 --- a/x-pack/plugins/file_upload/public/components/index_settings.js +++ b/x-pack/plugins/file_upload/public/components/index_settings.js @@ -118,6 +118,7 @@ export class IndexSettings extends Component { text: indexType, value: indexType, }))} + value={this.props.selectedIndexType} onChange={({ target }) => setSelectedIndexType(target.value)} /> diff --git a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js index 018d09a1b2f3..ae9f5e9ac815 100644 --- a/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js +++ b/x-pack/plugins/file_upload/public/components/json_upload_and_parse.js @@ -13,6 +13,7 @@ import { IndexSettings } from './index_settings'; import { getIndexPatternService } from '../kibana_services'; import { GeoJsonFilePicker } from './geojson_file_picker'; import { ImportCompleteView } from './import_complete_view'; +import { ES_FIELD_TYPES } from '../../../../../src/plugins/data/public'; const PHASE = { CONFIGURE: 'CONFIGURE', @@ -201,16 +202,21 @@ export class JsonUploadAndParse extends Component { }); }; - _onFileSelect = ({ features, geoFieldTypes, importer, indexName, previewCoverage }) => { + _onFileSelect = ({ features, hasPoints, hasShapes, importer, indexName, previewCoverage }) => { this._geojsonImporter = importer; + const geoFieldTypes = hasPoints + ? [ES_FIELD_TYPES.GEO_POINT, ES_FIELD_TYPES.GEO_SHAPE] + : [ES_FIELD_TYPES.GEO_SHAPE]; + const newState = { indexTypes: geoFieldTypes, indexName, }; - if (!this.state.selectedIndexType && geoFieldTypes.length) { + if (!this.state.selectedIndexType) { // auto select index type - newState.selectedIndexType = geoFieldTypes[0]; + newState.selectedIndexType = + hasPoints && !hasShapes ? ES_FIELD_TYPES.GEO_POINT : ES_FIELD_TYPES.GEO_SHAPE; } else if ( this.state.selectedIndexType && !geoFieldTypes.includes(this.state.selectedIndexType) @@ -274,6 +280,7 @@ export class JsonUploadAndParse extends Component { indexName={this.state.indexName} setIndexName={(indexName) => this.setState({ indexName })} indexTypes={this.state.indexTypes} + selectedIndexType={this.state.selectedIndexType} setSelectedIndexType={(selectedIndexType) => this.setState({ selectedIndexType })} setHasIndexErrors={(hasIndexErrors) => this.setState({ hasIndexErrors })} /> diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js index b91c23462250..98f14f3e5816 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js @@ -44,7 +44,8 @@ describe('previewFile', () => { expect(results).toEqual({ features: [], previewCoverage: 0, - geoFieldTypes: ['geo_shape'], + hasPoints: false, + hasShapes: false, }); }); @@ -53,7 +54,8 @@ describe('previewFile', () => { const results = await importer.previewFile(); expect(results).toEqual({ previewCoverage: 100, - geoFieldTypes: ['geo_point', 'geo_shape'], + hasPoints: true, + hasShapes: false, features: FEATURE_COLLECTION.features, }); }); @@ -88,7 +90,8 @@ describe('previewFile', () => { expect(results).toEqual({ previewCoverage: 100, - geoFieldTypes: ['geo_point', 'geo_shape'], + hasPoints: true, + hasShapes: false, features: FEATURE_COLLECTION.features, }); }); @@ -116,7 +119,8 @@ describe('previewFile', () => { expect(results).toEqual({ previewCoverage: 100, - geoFieldTypes: ['geo_point', 'geo_shape'], + hasPoints: true, + hasShapes: false, features: FEATURE_COLLECTION.features, }); }); @@ -138,7 +142,8 @@ describe('previewFile', () => { expect(results).toEqual({ previewCoverage: 100, - geoFieldTypes: ['geo_shape'], + hasPoints: false, + hasShapes: false, features: [], }); }); @@ -165,7 +170,8 @@ describe('previewFile', () => { expect(results).toEqual({ previewCoverage: 100, - geoFieldTypes: ['geo_shape'], + hasPoints: false, + hasShapes: false, features: [], }); }); diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts index 2cb2a10b9560..3294655916b5 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts @@ -27,6 +27,13 @@ import { ImportFailure, ImportResponse, MB } from '../../../common'; const IMPORT_CHUNK_SIZE_MB = 10 * MB; export const GEOJSON_FILE_TYPES = ['.json', '.geojson']; +export interface GeoJsonPreview { + features: Feature[]; + hasPoints: boolean; + hasShapes: boolean; + previewCoverage: number; +} + export class GeoJsonImporter extends Importer { private _file: File; private _isActive = true; @@ -52,20 +59,19 @@ export class GeoJsonImporter extends Importer { this._isActive = false; } - public async previewFile( - rowLimit?: number, - sizeLimit?: number - ): Promise<{ features: Feature[]; geoFieldTypes: string[]; previewCoverage: number }> { + public async previewFile(rowLimit?: number, sizeLimit?: number): Promise { await this._readUntil(rowLimit, sizeLimit); return { features: [...this._features], previewCoverage: this._hasNext ? Math.round((this._unimportedBytesProcessed / this._file.size) * 100) : 100, - geoFieldTypes: - this._geometryTypesMap.has('Point') || this._geometryTypesMap.has('MultiPoint') - ? [ES_FIELD_TYPES.GEO_POINT, ES_FIELD_TYPES.GEO_SHAPE] - : [ES_FIELD_TYPES.GEO_SHAPE], + hasPoints: this._geometryTypesMap.has('Point') || this._geometryTypesMap.has('MultiPoint'), + hasShapes: + this._geometryTypesMap.has('LineString') || + this._geometryTypesMap.has('MultiLineString') || + this._geometryTypesMap.has('Polygon') || + this._geometryTypesMap.has('MultiPolygon'), }; } diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/index.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/index.ts index fccf1a5d9ae2..b5f6845e2832 100644 --- a/x-pack/plugins/file_upload/public/importer/geojson_importer/index.ts +++ b/x-pack/plugins/file_upload/public/importer/geojson_importer/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export { GeoJsonImporter, GEOJSON_FILE_TYPES } from './geojson_importer'; +export { GeoJsonImporter, GeoJsonPreview, GEOJSON_FILE_TYPES } from './geojson_importer'; From 5968394ec4e885e3b45f7ea407b9bc40ea36d412 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 2 Mar 2021 16:38:27 -0700 Subject: [PATCH 26/26] fix wizard onError callback to not advance to next step --- .../maps/public/classes/layers/file_upload_wizard/wizard.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx b/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx index 82a5298d5b42..30e3317a3a3c 100644 --- a/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx +++ b/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx @@ -142,7 +142,8 @@ export class ClientFileCreateSourceEditor extends Component