From 875f3f3593c34b50c94f3fad07f6301079d2a7f2 Mon Sep 17 00:00:00 2001 From: Sander Philipse <94373878+sphilipse@users.noreply.github.com> Date: Wed, 3 May 2023 16:41:12 +0200 Subject: [PATCH 01/11] [Enterprise Search] Promote Postgresql and MSSQL connectors to native (#156277) ## Summary This adds PostgreSQL and MSSQL as native connectors. --- .../common/connectors/connectors.ts | 40 +- .../common/connectors/native_connectors.ts | 468 ++++++++++++++++++ .../common/types/connectors.ts | 2 + .../__mocks__/search_indices.mock.ts | 4 +- .../__mocks__/sync_job.mock.ts | 1 + .../__mocks__/view_index.mock.ts | 4 +- .../connector_configuration_logic.test.ts | 46 ++ .../connector_configuration_logic.ts | 5 +- .../sync_jobs/sync_job_flyout.tsx | 2 +- .../sync_jobs/sync_jobs_view_logic.test.ts | 1 + .../server/lib/connectors/start_sync.test.ts | 2 + .../server/lib/connectors/start_sync.ts | 1 + 12 files changed, 550 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/enterprise_search/common/connectors/connectors.ts b/x-pack/plugins/enterprise_search/common/connectors/connectors.ts index 953c49b493b3..38e445c50bef 100644 --- a/x-pack/plugins/enterprise_search/common/connectors/connectors.ts +++ b/x-pack/plugins/enterprise_search/common/connectors/connectors.ts @@ -37,6 +37,26 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ }), serviceType: 'mysql', }, + { + iconPath: 'mssql.svg', + isBeta: true, + isNative: true, + keywords: ['mssql', 'microsoft', 'sql', 'database', 'connector'], + name: i18n.translate('xpack.enterpriseSearch.content.nativeConnectors.microsoftSQL.name', { + defaultMessage: 'Microsoft SQL', + }), + serviceType: 'mssql', + }, + { + iconPath: 'postgresql.svg', + isBeta: true, + isNative: true, + keywords: ['postgresql', 'sql', 'database', 'connector'], + name: i18n.translate('xpack.enterpriseSearch.content.nativeConnectors.postgresql.name', { + defaultMessage: 'Postgresql', + }), + serviceType: 'postgresql', + }, { iconPath: 'azure_blob_storage.svg', isBeta: true, @@ -57,16 +77,6 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ }), serviceType: 'google_cloud_storage', }, - { - iconPath: 'mssql.svg', - isBeta: true, - isNative: false, - keywords: ['mssql', 'microsoft', 'sql', 'database', 'connector'], - name: i18n.translate('xpack.enterpriseSearch.content.nativeConnectors.microsoftSQL.name', { - defaultMessage: 'Microsoft SQL', - }), - serviceType: 'mssql', - }, { iconPath: 'network_drive.svg', isBeta: true, @@ -87,16 +97,6 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ }), serviceType: 'oracle', }, - { - iconPath: 'postgresql.svg', - isBeta: true, - isNative: false, - keywords: ['postgresql', 'sql', 'database', 'connector'], - name: i18n.translate('xpack.enterpriseSearch.content.nativeConnectors.postgresql.name', { - defaultMessage: 'Postgresql', - }), - serviceType: 'postgresql', - }, { iconPath: 's3.svg', isBeta: true, diff --git a/x-pack/plugins/enterprise_search/common/connectors/native_connectors.ts b/x-pack/plugins/enterprise_search/common/connectors/native_connectors.ts index a130d1260d5c..cbdc221eb629 100644 --- a/x-pack/plugins/enterprise_search/common/connectors/native_connectors.ts +++ b/x-pack/plugins/enterprise_search/common/connectors/native_connectors.ts @@ -146,6 +146,260 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record; started_at: string | null; status: SyncStatus; + total_document_count: number | null; trigger_method: TriggerMethod; worker_hostname: string | null; } diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/search_indices.mock.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/search_indices.mock.ts index 0f864125659f..b0b5d4aa555a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/search_indices.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/search_indices.mock.ts @@ -39,7 +39,6 @@ export const indices: ElasticsearchIndexWithIngestion[] = [ default_value: '', depends_on: [], display: DisplayType.TEXTBOX, - key: 'foo', label: 'bar', options: [], order: 1, @@ -48,6 +47,7 @@ export const indices: ElasticsearchIndexWithIngestion[] = [ tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'barbar', }, }, @@ -151,7 +151,6 @@ export const indices: ElasticsearchIndexWithIngestion[] = [ default_value: '', depends_on: [], display: DisplayType.TEXTBOX, - key: 'foo', label: 'bar', options: [], order: 1, @@ -160,6 +159,7 @@ export const indices: ElasticsearchIndexWithIngestion[] = [ tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'barbar', }, }, diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/sync_job.mock.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/sync_job.mock.ts index 04a3908f5d5b..e09a221df89b 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/sync_job.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/sync_job.mock.ts @@ -33,6 +33,7 @@ export const syncJob: ConnectorSyncJob = { metadata: {}, started_at: '2022-09-05T14:59:39.816+00:00', status: SyncStatus.COMPLETED, + total_document_count: null, trigger_method: TriggerMethod.ON_DEMAND, worker_hostname: 'hostname_fake', }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/view_index.mock.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/view_index.mock.ts index 4e2a0ceda0fb..bc4d669f699a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/view_index.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/view_index.mock.ts @@ -49,7 +49,6 @@ export const connectorIndex: ConnectorViewIndex = { default_value: '', depends_on: [], display: DisplayType.TEXTBOX, - key: 'foo', label: 'bar', options: [], order: 1, @@ -58,6 +57,7 @@ export const connectorIndex: ConnectorViewIndex = { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'barbar', }, }, @@ -165,7 +165,6 @@ export const crawlerIndex: CrawlerViewIndex = { default_value: '', depends_on: [], display: DisplayType.TEXTBOX, - key: 'foo', label: 'bar', options: [], order: 1, @@ -174,6 +173,7 @@ export const crawlerIndex: CrawlerViewIndex = { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'barbar', }, }, diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.test.ts index 2211f64645c3..4092050b3b51 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.test.ts @@ -63,6 +63,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'oldBar', }, }, @@ -83,6 +84,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'oldBar', }, }, @@ -101,6 +103,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'oldBar', }, @@ -121,6 +124,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'fourthBar', }, }); @@ -139,6 +143,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'fourthBar', }, }, @@ -157,6 +162,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'fourthBar', }, @@ -178,6 +184,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'foofoo', }, password: { @@ -192,6 +199,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'fourthBar', }, restricted: { @@ -206,6 +214,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: ['advanced'], + validations: [], value: 'I am restricted', }, shownDependent1: { @@ -220,6 +229,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should appear (one dependency)', }, shownDependent2: { @@ -237,6 +247,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should appear (multiple dependencies)', }, hiddenDependent1: { @@ -251,6 +262,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should hide (one dependency)', }, hiddenDependent2: { @@ -268,6 +280,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should hide (multiple dependencies)', }, }); @@ -284,6 +297,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'foofoo', }, password: { @@ -298,6 +312,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'fourthBar', }, restricted: { @@ -312,6 +327,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: ['advanced'], + validations: [], value: 'I am restricted', }, shownDependent1: { @@ -326,6 +342,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should appear (one dependency)', }, shownDependent2: { @@ -343,6 +360,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should appear (multiple dependencies)', }, hiddenDependent1: { @@ -357,6 +375,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should hide (one dependency)', }, hiddenDependent2: { @@ -374,6 +393,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should hide (multiple dependencies)', }, }); @@ -409,6 +429,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'foofoo', }, password: { @@ -423,6 +444,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'fourthBar', }, restricted: { @@ -437,6 +459,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: ['advanced'], + validations: [], value: 'I am restricted', }, shownDependent1: { @@ -451,6 +474,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should appear (one dependency)', }, shownDependent2: { @@ -468,6 +492,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should appear (multiple dependencies)', }, hiddenDependent1: { @@ -482,6 +507,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should hide (one dependency)', }, hiddenDependent2: { @@ -499,6 +525,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should hide (multiple dependencies)', }, }, @@ -517,6 +544,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'foofoo', }, @@ -534,6 +562,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'fourthBar', }, @@ -551,6 +580,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'I should appear (one dependency)', }, @@ -571,6 +601,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'I should appear (multiple dependencies)', }, @@ -588,6 +619,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'fafa', }, password: { @@ -602,6 +634,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'fourthBar', }, restricted: { @@ -616,6 +649,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: ['advanced'], + validations: [], value: 'I am restricted', }, shownDependent1: { @@ -630,6 +664,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should appear (one dependency)', }, shownDependent2: { @@ -647,6 +682,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should appear (multiple dependencies)', }, hiddenDependent1: { @@ -661,6 +697,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should hide (one dependency)', }, hiddenDependent2: { @@ -678,6 +715,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], value: 'I should hide (multiple dependencies)', }, }, @@ -696,6 +734,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'fafa', }, @@ -713,6 +752,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'fourthBar', }, @@ -730,6 +770,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'I should hide (one dependency)', }, @@ -750,6 +791,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'I should hide (multiple dependencies)', }, @@ -778,6 +820,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'barbar', }, @@ -818,6 +861,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'barbar', }, @@ -843,6 +887,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'barbar', }, @@ -869,6 +914,7 @@ describe('ConnectorConfigurationLogic', () => { tooltip: '', type: FieldType.STRING, ui_restrictions: [], + validations: [], validation_errors: [], value: 'Barbara', }, diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.ts index 065fe42d5bb0..1aec514c0cd0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.ts @@ -73,6 +73,7 @@ export interface ConfigEntry { type: FieldType; ui_restrictions: string[]; validation_errors: string[]; + validations?: string[]; value: string | number | boolean | null; } @@ -155,7 +156,7 @@ function validIntInput(value: string | number | boolean | null): boolean { // reject non integers (including x.0 floats), but don't validate if empty return (value !== null || value !== '') && (isNaN(Number(value)) || - !Number.isSafeInteger(value) || + !Number.isSafeInteger(Number(value)) || ensureStringType(value).indexOf('.') >= 0) ? false : true; @@ -328,6 +329,7 @@ export const ConnectorConfigurationLogic = kea< type, // eslint-disable-next-line @typescript-eslint/naming-convention ui_restrictions, + validations, value, } ) => ({ @@ -344,6 +346,7 @@ export const ConnectorConfigurationLogic = kea< tooltip, type, ui_restrictions, + validations: validations ?? [], value: ensureCorrectTyping(type, value), }, }), diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs/sync_job_flyout.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs/sync_job_flyout.tsx index b3257fa6455e..0893b6e99d93 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs/sync_job_flyout.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs/sync_job_flyout.tsx @@ -75,7 +75,7 @@ export const SyncJobFlyout: React.FC = ({ onClose, syncJob } diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs/sync_jobs_view_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs/sync_jobs_view_logic.test.ts index 1b965632194b..41ac136437ef 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs/sync_jobs_view_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs/sync_jobs_view_logic.test.ts @@ -80,6 +80,7 @@ describe('SyncJobsViewLogic', () => { metadata: {}, started_at: '2022-09-05T14:59:39.816+00:00', status: SyncStatus.COMPLETED, + total_document_count: null, trigger_method: TriggerMethod.ON_DEMAND, worker_hostname: 'hostname_fake', }; diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.test.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.test.ts index f9dedfe06917..43c7a81ac527 100644 --- a/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.test.ts @@ -81,6 +81,7 @@ describe('startSync lib function', () => { metadata: {}, started_at: null, status: SyncStatus.PENDING, + total_document_count: null, trigger_method: TriggerMethod.ON_DEMAND, worker_hostname: null, }, @@ -143,6 +144,7 @@ describe('startSync lib function', () => { metadata: {}, started_at: null, status: SyncStatus.PENDING, + total_document_count: null, trigger_method: TriggerMethod.ON_DEMAND, worker_hostname: null, }, diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.ts index 0028b7185e86..c2c6e0a9995f 100644 --- a/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.ts +++ b/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.ts @@ -76,6 +76,7 @@ export const startConnectorSync = async ( metadata: {}, started_at: null, status: SyncStatus.PENDING, + total_document_count: null, trigger_method: TriggerMethod.ON_DEMAND, worker_hostname: null, }, From ff65ca42f2f09589b09a8e61abe22a4a69885170 Mon Sep 17 00:00:00 2001 From: Jatin Kathuria Date: Wed, 3 May 2023 16:57:11 +0200 Subject: [PATCH 02/11] [Security Solution] [Fix] Alert Page Controls do not recover from invalid query. (#156542) ## Summary This PR handles #156016 . Previously, if user supplied an invalid kql or lucene query, Alert Page controls will go in error state and not recover until user reloaded the page or navigated away and back to the Alert Page. This PR prevents Alert Page Controls going in that error state. | Before | After | |--|--| |