From 22911c1828f40160cf3a2935300aec18c11b56e9 Mon Sep 17 00:00:00 2001 From: Dmitrii Shevchenko Date: Tue, 3 Dec 2024 13:11:24 +0100 Subject: [PATCH 01/51] [Security Solution] Skip isCustomized calculation when the feature flag is off (#201825) **Resolves: https://github.com/elastic/kibana/issues/201632** ## Summary When the rule customization feature flag is disabled, we should always return `isCustomized: false`, regardless of any changes introduced to a rule. This ensures that we do not accidentally mark prebuilt rules as customized in 8.16 with the feature flag off. For more details, refer to the related issue: https://github.com/elastic/kibana/issues/201632 ### Main Changes - The primary change in this PR is encapsulated in the `calculateIsCustomized` function - Other changes involve passing the feature flag to this function - Added integration tests to cover all API CRUD operations that can be performed with rules --- .../ftr_security_serverless_configs.yml | 3 +- .buildkite/ftr_security_stateful_configs.yml | 3 +- .../logic/bulk_actions/bulk_edit_rules.ts | 14 +- ...on_rules_client.create_custom_rule.test.ts | 1 + ..._rules_client.create_prebuilt_rule.test.ts | 1 + ...detection_rules_client.delete_rule.test.ts | 1 + ...detection_rules_client.import_rule.test.ts | 1 + ...etection_rules_client.import_rules.test.ts | 1 + .../detection_rules_client.patch_rule.test.ts | 1 + .../detection_rules_client.ts | 6 + ...detection_rules_client.update_rule.test.ts | 1 + ...rules_client.upgrade_prebuilt_rule.test.ts | 1 + .../mergers/apply_rule_patch.test.ts | 22 ++ .../mergers/apply_rule_patch.ts | 3 + .../mergers/apply_rule_update.ts | 3 + .../rule_source/calculate_is_customized.ts | 20 +- .../rule_source/calculate_rule_source.test.ts | 25 ++ .../rule_source/calculate_rule_source.ts | 8 +- .../methods/import_rule.ts | 3 + .../methods/patch_rule.ts | 3 + .../methods/update_rule.ts | 3 + .../methods/upgrade_prebuilt_rule.ts | 3 + .../calculate_rule_source_for_import.test.ts | 4 + .../calculate_rule_source_for_import.ts | 3 + .../calculate_rule_source_from_asset.test.ts | 4 + .../calculate_rule_source_from_asset.ts | 10 +- .../rule_source_importer.test.ts | 2 + .../rule_source_importer.ts | 2 + .../server/request_context_factory.ts | 1 + .../configs/ess.config.ts | 25 ++ .../configs/serverless.config.ts | 17 ++ .../customization_disabled/index.ts | 14 ++ .../is_customized_calculation.ts | 218 ++++++++++++++++++ .../configs/ess.config.ts | 2 +- .../configs/serverless.config.ts | 2 +- .../import_rules.ts | 0 .../index.ts | 2 +- .../is_customized_calculation.ts | 0 .../rules_export.ts | 0 .../patch_rules.ts | 28 +-- 40 files changed, 425 insertions(+), 36 deletions(-) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/ess.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/serverless.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/index.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/is_customized_calculation.ts rename x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/{trial_license_complete_tier => customization_enabled}/configs/ess.config.ts (91%) rename x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/{trial_license_complete_tier => customization_enabled}/configs/serverless.config.ts (84%) rename x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/{trial_license_complete_tier => customization_enabled}/import_rules.ts (100%) rename x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/{trial_license_complete_tier => customization_enabled}/index.ts (94%) rename x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/{trial_license_complete_tier => customization_enabled}/is_customized_calculation.ts (100%) rename x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/{trial_license_complete_tier => customization_enabled}/rules_export.ts (100%) diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml index 920ecce349356..74d82d40c8bce 100644 --- a/.buildkite/ftr_security_serverless_configs.yml +++ b/.buildkite/ftr_security_serverless_configs.yml @@ -76,7 +76,8 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/large_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/serverless.config.ts diff --git a/.buildkite/ftr_security_stateful_configs.yml b/.buildkite/ftr_security_stateful_configs.yml index f7caacba05e1b..46c6f356b3ae4 100644 --- a/.buildkite/ftr_security_stateful_configs.yml +++ b/.buildkite/ftr_security_stateful_configs.yml @@ -58,7 +58,8 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/large_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/ess.config.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/bulk_edit_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/bulk_edit_rules.ts index bee44a3600f97..b9c97c2412b1a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/bulk_edit_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/bulk_edit_rules.ts @@ -92,14 +92,20 @@ export const bulkEditRules = async ({ params: modifiedParams, }; const ruleResponse = convertAlertingRuleToRuleResponse(updatedRule); + let isCustomized = false; + if (ruleResponse.immutable === true) { + isCustomized = calculateIsCustomized({ + baseRule: baseVersionsMap.get(ruleResponse.rule_id), + nextRule: ruleResponse, + isRuleCustomizationEnabled: experimentalFeatures.prebuiltRulesCustomizationEnabled, + }); + } + const ruleSource = ruleResponse.immutable === true ? { type: 'external' as const, - isCustomized: calculateIsCustomized( - baseVersionsMap.get(ruleResponse.rule_id), - ruleResponse - ), + isCustomized, } : { type: 'internal' as const, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.create_custom_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.create_custom_rule.test.ts index af3f85f3e1345..b07f95f74d2f9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.create_custom_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.create_custom_rule.test.ts @@ -51,6 +51,7 @@ describe('DetectionRulesClient.createCustomRule', () => { rulesClient, mlAuthz, savedObjectsClient, + isRuleCustomizationEnabled: true, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.create_prebuilt_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.create_prebuilt_rule.test.ts index 4caeb4bd9b940..6e1e75855b6a0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.create_prebuilt_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.create_prebuilt_rule.test.ts @@ -44,6 +44,7 @@ describe('DetectionRulesClient.createPrebuiltRule', () => { rulesClient, mlAuthz, savedObjectsClient, + isRuleCustomizationEnabled: true, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.delete_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.delete_rule.test.ts index ea5d1453753c3..a4afe5abadb6b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.delete_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.delete_rule.test.ts @@ -30,6 +30,7 @@ describe('DetectionRulesClient.deleteRule', () => { rulesClient, mlAuthz, savedObjectsClient, + isRuleCustomizationEnabled: true, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts index 4300b17b3be80..5f8bfb5577740 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts @@ -51,6 +51,7 @@ describe('DetectionRulesClient.importRule', () => { rulesClient, mlAuthz, savedObjectsClient, + isRuleCustomizationEnabled: true, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts index 58b1385dda09c..3dfd859fb0b7b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts @@ -32,6 +32,7 @@ describe('detectionRulesClient.importRules', () => { rulesClient: rulesClientMock.create(), mlAuthz: buildMlAuthz(), savedObjectsClient: savedObjectsClientMock.create(), + isRuleCustomizationEnabled: true, }); (checkRuleExceptionReferences as jest.Mock).mockReturnValue([[], []]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.patch_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.patch_rule.test.ts index 448df6b581a3b..18fe3c54da7e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.patch_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.patch_rule.test.ts @@ -50,6 +50,7 @@ describe('DetectionRulesClient.patchRule', () => { rulesClient, mlAuthz, savedObjectsClient, + isRuleCustomizationEnabled: true, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts index fb6f5c4a03c1f..57c8f12c09aa9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts @@ -38,6 +38,7 @@ interface DetectionRulesClientParams { rulesClient: RulesClient; savedObjectsClient: SavedObjectsClientContract; mlAuthz: MlAuthz; + isRuleCustomizationEnabled: boolean; } export const createDetectionRulesClient = ({ @@ -45,6 +46,7 @@ export const createDetectionRulesClient = ({ rulesClient, mlAuthz, savedObjectsClient, + isRuleCustomizationEnabled, }: DetectionRulesClientParams): IDetectionRulesClient => { const prebuiltRuleAssetClient = createPrebuiltRuleAssetsClient(savedObjectsClient); @@ -89,6 +91,7 @@ export const createDetectionRulesClient = ({ prebuiltRuleAssetClient, mlAuthz, ruleUpdate, + isRuleCustomizationEnabled, }); }); }, @@ -101,6 +104,7 @@ export const createDetectionRulesClient = ({ prebuiltRuleAssetClient, mlAuthz, rulePatch, + isRuleCustomizationEnabled, }); }); }, @@ -119,6 +123,7 @@ export const createDetectionRulesClient = ({ ruleAsset, mlAuthz, prebuiltRuleAssetClient, + isRuleCustomizationEnabled, }); }); }, @@ -131,6 +136,7 @@ export const createDetectionRulesClient = ({ importRulePayload: args, mlAuthz, prebuiltRuleAssetClient, + isRuleCustomizationEnabled, }); }); }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.update_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.update_rule.test.ts index cbd0fb1fe3680..64c01ab395529 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.update_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.update_rule.test.ts @@ -50,6 +50,7 @@ describe('DetectionRulesClient.updateRule', () => { rulesClient, mlAuthz, savedObjectsClient, + isRuleCustomizationEnabled: true, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.upgrade_prebuilt_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.upgrade_prebuilt_rule.test.ts index acdb7b9653930..ed186abe5a7c3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.upgrade_prebuilt_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.upgrade_prebuilt_rule.test.ts @@ -48,6 +48,7 @@ describe('DetectionRulesClient.upgradePrebuiltRule', () => { rulesClient, mlAuthz, savedObjectsClient, + isRuleCustomizationEnabled: true, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_patch.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_patch.test.ts index abf90c3f4dfc4..0d780c07aac19 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_patch.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_patch.test.ts @@ -37,6 +37,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -65,6 +66,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -94,6 +96,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }) ).rejects.toThrowError( 'event_category_override: Expected string, received number, tiebreaker_field: Expected string, received number, timestamp_field: Expected string, received number' @@ -119,6 +122,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }) ).rejects.toThrowError('alert_suppression.group_by: Expected array, received string'); }); @@ -134,6 +138,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -154,6 +159,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }) ).rejects.toThrowError( 'threat_query: Expected string, received number, threat_indicator_path: Expected string, received number' @@ -170,6 +176,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -190,6 +197,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }) ).rejects.toThrowError( "index.0: Expected string, received number, language: Invalid enum value. Expected 'kuery' | 'lucene', received 'non-language'" @@ -206,6 +214,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -226,6 +235,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }) ).rejects.toThrowError( "index.0: Expected string, received number, language: Invalid enum value. Expected 'kuery' | 'lucene', received 'non-language'" @@ -244,6 +254,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -268,6 +279,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }) ).rejects.toThrowError('threshold.value: Expected number, received string'); }); @@ -285,6 +297,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -308,6 +321,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -330,6 +344,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -354,6 +369,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -376,6 +392,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -394,6 +411,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }) ).rejects.toThrowError('anomaly_threshold: Expected number, received string'); }); @@ -410,6 +428,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( @@ -432,6 +451,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ @@ -450,6 +470,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }) ).rejects.toThrowError('new_terms_fields: Expected array, received string'); }); @@ -472,6 +493,7 @@ describe('applyRulePatch', () => { rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled: true, }); expect(patchedRule).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_patch.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_patch.ts index 9f5b167322491..aaaab474c2fbe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_patch.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_patch.ts @@ -51,6 +51,7 @@ interface ApplyRulePatchProps { prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; existingRule: RuleResponse; rulePatch: PatchRuleRequestBody; + isRuleCustomizationEnabled: boolean; } // eslint-disable-next-line complexity @@ -58,6 +59,7 @@ export const applyRulePatch = async ({ rulePatch, existingRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled, }: ApplyRulePatchProps): Promise => { const typeSpecificParams = patchTypeSpecificParams(rulePatch, existingRule); @@ -122,6 +124,7 @@ export const applyRulePatch = async ({ nextRule.rule_source = await calculateRuleSource({ rule: nextRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled, }); return nextRule; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts index b911e66a1fc45..850924d3cd04d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts @@ -17,12 +17,14 @@ interface ApplyRuleUpdateProps { prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; existingRule: RuleResponse; ruleUpdate: RuleUpdateProps; + isRuleCustomizationEnabled: boolean; } export const applyRuleUpdate = async ({ prebuiltRuleAssetClient, existingRule, ruleUpdate, + isRuleCustomizationEnabled, }: ApplyRuleUpdateProps): Promise => { const nextRule: RuleResponse = { ...applyRuleDefaults(ruleUpdate), @@ -46,6 +48,7 @@ export const applyRuleUpdate = async ({ nextRule.rule_source = await calculateRuleSource({ rule: nextRule, prebuiltRuleAssetClient, + isRuleCustomizationEnabled, }); return nextRule; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts index 92c7d941dd7f1..a94506486cc53 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts @@ -12,10 +12,22 @@ import { calculateRuleFieldsDiff } from '../../../../../prebuilt_rules/logic/dif import { convertRuleToDiffable } from '../../../../../../../../common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable'; import { convertPrebuiltRuleAssetToRuleResponse } from '../../converters/convert_prebuilt_rule_asset_to_rule_response'; -export function calculateIsCustomized( - baseRule: PrebuiltRuleAsset | undefined, - nextRule: RuleResponse -) { +interface CalculateIsCustomizedArgs { + baseRule: PrebuiltRuleAsset | undefined; + nextRule: RuleResponse; + isRuleCustomizationEnabled: boolean; +} + +export function calculateIsCustomized({ + baseRule, + nextRule, + isRuleCustomizationEnabled, +}: CalculateIsCustomizedArgs) { + if (!isRuleCustomizationEnabled) { + // We don't want to accidentally mark rules as customized when customization is disabled. + return false; + } + if (baseRule == null) { // If the base version is missing, we consider the rule to be customized return true; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.test.ts index e44c69d2705d5..d0b9f722458a1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.test.ts @@ -43,6 +43,7 @@ describe('calculateRuleSource', () => { const result = await calculateRuleSource({ prebuiltRuleAssetClient, rule, + isRuleCustomizationEnabled: true, }); expect(result).toEqual({ type: 'internal', @@ -59,6 +60,7 @@ describe('calculateRuleSource', () => { const result = await calculateRuleSource({ prebuiltRuleAssetClient, rule, + isRuleCustomizationEnabled: true, }); expect(result).toEqual( expect.objectContaining({ @@ -79,6 +81,7 @@ describe('calculateRuleSource', () => { const result = await calculateRuleSource({ prebuiltRuleAssetClient, rule, + isRuleCustomizationEnabled: true, }); expect(result).toEqual( expect.objectContaining({ @@ -101,6 +104,28 @@ describe('calculateRuleSource', () => { const result = await calculateRuleSource({ prebuiltRuleAssetClient, rule, + isRuleCustomizationEnabled: true, + }); + expect(result).toEqual( + expect.objectContaining({ + type: 'external', + is_customized: false, + }) + ); + }); + + it('returns is_customized false when the rule is customized but customization is disabled', async () => { + const rule = getSampleRule(); + rule.immutable = true; + rule.name = 'Updated name'; + + const baseRule = getSampleRuleAsset(); + prebuiltRuleAssetClient.fetchAssetsByVersion.mockResolvedValueOnce([baseRule]); + + const result = await calculateRuleSource({ + prebuiltRuleAssetClient, + rule, + isRuleCustomizationEnabled: false, }); expect(result).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.ts index 742cd20544a60..6c3d2419159a6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.ts @@ -16,11 +16,13 @@ import { calculateIsCustomized } from './calculate_is_customized'; interface CalculateRuleSourceProps { prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; rule: RuleResponse; + isRuleCustomizationEnabled: boolean; } export async function calculateRuleSource({ prebuiltRuleAssetClient, rule, + isRuleCustomizationEnabled, }: CalculateRuleSourceProps): Promise { if (rule.immutable) { // This is a prebuilt rule and, despite the name, they are not immutable. So @@ -33,7 +35,11 @@ export async function calculateRuleSource({ ]); const baseRule: PrebuiltRuleAsset | undefined = prebuiltRulesResponse.at(0); - const isCustomized = calculateIsCustomized(baseRule, rule); + const isCustomized = calculateIsCustomized({ + baseRule, + nextRule: rule, + isRuleCustomizationEnabled, + }); return { type: 'external', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts index dd57e66c41a64..16fda89391ab9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts @@ -26,6 +26,7 @@ interface ImportRuleOptions { prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; importRulePayload: ImportRuleArgs; mlAuthz: MlAuthz; + isRuleCustomizationEnabled: boolean; } export const importRule = async ({ @@ -34,6 +35,7 @@ export const importRule = async ({ importRulePayload, prebuiltRuleAssetClient, mlAuthz, + isRuleCustomizationEnabled, }: ImportRuleOptions): Promise => { const { ruleToImport, overwriteRules, overrideFields, allowMissingConnectorSecrets } = importRulePayload; @@ -60,6 +62,7 @@ export const importRule = async ({ prebuiltRuleAssetClient, existingRule, ruleUpdate: rule, + isRuleCustomizationEnabled, }); // applyRuleUpdate prefers the existing rule's values for `rule_source` and `immutable`, but we want to use the importing rule's calculated values ruleWithUpdates = { ...ruleWithUpdates, ...overrideFields }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/patch_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/patch_rule.ts index 113576e8d02e2..0b22ea39eaef4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/patch_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/patch_rule.ts @@ -28,6 +28,7 @@ interface PatchRuleOptions { prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; rulePatch: RulePatchProps; mlAuthz: MlAuthz; + isRuleCustomizationEnabled: boolean; } export const patchRule = async ({ @@ -36,6 +37,7 @@ export const patchRule = async ({ prebuiltRuleAssetClient, rulePatch, mlAuthz, + isRuleCustomizationEnabled, }: PatchRuleOptions): Promise => { const { rule_id: ruleId, id } = rulePatch; @@ -58,6 +60,7 @@ export const patchRule = async ({ prebuiltRuleAssetClient, existingRule, rulePatch, + isRuleCustomizationEnabled, }); const patchedInternalRule = await rulesClient.update({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/update_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/update_rule.ts index 8fd7f7a89dcb7..3d465d44fdc1d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/update_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/update_rule.ts @@ -27,6 +27,7 @@ interface UpdateRuleArguments { prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; ruleUpdate: RuleUpdateProps; mlAuthz: MlAuthz; + isRuleCustomizationEnabled: boolean; } export const updateRule = async ({ @@ -35,6 +36,7 @@ export const updateRule = async ({ prebuiltRuleAssetClient, ruleUpdate, mlAuthz, + isRuleCustomizationEnabled, }: UpdateRuleArguments): Promise => { const { rule_id: ruleId, id } = ruleUpdate; @@ -57,6 +59,7 @@ export const updateRule = async ({ prebuiltRuleAssetClient, existingRule, ruleUpdate, + isRuleCustomizationEnabled, }); const updatedRule = await rulesClient.update({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/upgrade_prebuilt_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/upgrade_prebuilt_rule.ts index 64486bed14304..dff7c8a333ca7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/upgrade_prebuilt_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/upgrade_prebuilt_rule.ts @@ -25,12 +25,14 @@ export const upgradePrebuiltRule = async ({ ruleAsset, mlAuthz, prebuiltRuleAssetClient, + isRuleCustomizationEnabled, }: { actionsClient: ActionsClient; rulesClient: RulesClient; ruleAsset: PrebuiltRuleAsset; mlAuthz: MlAuthz; prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; + isRuleCustomizationEnabled: boolean; }): Promise => { await validateMlAuth(mlAuthz, ruleAsset.type); @@ -73,6 +75,7 @@ export const upgradePrebuiltRule = async ({ prebuiltRuleAssetClient, existingRule, ruleUpdate: ruleAsset, + isRuleCustomizationEnabled, }); const updatedInternalRule = await rulesClient.update({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts index e3bfb75c6a88d..b1952ff6948c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -15,6 +15,7 @@ describe('calculateRuleSourceForImport', () => { rule: getRulesSchemaMock(), prebuiltRuleAssetsByRuleId: {}, isKnownPrebuiltRule: false, + isRuleCustomizationEnabled: true, }); expect(result).toEqual({ @@ -33,6 +34,7 @@ describe('calculateRuleSourceForImport', () => { rule, prebuiltRuleAssetsByRuleId: {}, isKnownPrebuiltRule: true, + isRuleCustomizationEnabled: true, }); expect(result).toEqual({ @@ -53,6 +55,7 @@ describe('calculateRuleSourceForImport', () => { rule, prebuiltRuleAssetsByRuleId, isKnownPrebuiltRule: true, + isRuleCustomizationEnabled: true, }); expect(result).toEqual({ @@ -73,6 +76,7 @@ describe('calculateRuleSourceForImport', () => { rule, prebuiltRuleAssetsByRuleId, isKnownPrebuiltRule: true, + isRuleCustomizationEnabled: true, }); expect(result).toEqual({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 133566a7b776b..32a1f622b08d8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -28,10 +28,12 @@ export const calculateRuleSourceForImport = ({ rule, prebuiltRuleAssetsByRuleId, isKnownPrebuiltRule, + isRuleCustomizationEnabled, }: { rule: ValidatedRuleToImport; prebuiltRuleAssetsByRuleId: Record; isKnownPrebuiltRule: boolean; + isRuleCustomizationEnabled: boolean; }): { ruleSource: RuleSource; immutable: boolean } => { const assetWithMatchingVersion = prebuiltRuleAssetsByRuleId[rule.rule_id]; // We convert here so that RuleSource calculation can @@ -43,6 +45,7 @@ export const calculateRuleSourceForImport = ({ rule: ruleResponseForImport, assetWithMatchingVersion, isKnownPrebuiltRule, + isRuleCustomizationEnabled, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts index 9a2f68479fdea..099c0f41f1720 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts @@ -15,6 +15,7 @@ describe('calculateRuleSourceFromAsset', () => { rule: getRulesSchemaMock(), assetWithMatchingVersion: undefined, isKnownPrebuiltRule: false, + isRuleCustomizationEnabled: true, }); expect(result).toEqual({ @@ -28,6 +29,7 @@ describe('calculateRuleSourceFromAsset', () => { rule: ruleToImport, assetWithMatchingVersion: undefined, isKnownPrebuiltRule: true, + isRuleCustomizationEnabled: true, }); expect(result).toEqual({ @@ -47,6 +49,7 @@ describe('calculateRuleSourceFromAsset', () => { // no other overwrites -> no differences }), isKnownPrebuiltRule: true, + isRuleCustomizationEnabled: true, }); expect(result).toEqual({ @@ -65,6 +68,7 @@ describe('calculateRuleSourceFromAsset', () => { name: 'Customized name', // mock a customization }), isKnownPrebuiltRule: true, + isRuleCustomizationEnabled: true, }); expect(result).toEqual({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts index 4f0caf9b10056..33063bedb11b2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts @@ -24,10 +24,12 @@ export const calculateRuleSourceFromAsset = ({ rule, assetWithMatchingVersion, isKnownPrebuiltRule, + isRuleCustomizationEnabled, }: { rule: RuleResponse; assetWithMatchingVersion: PrebuiltRuleAsset | undefined; isKnownPrebuiltRule: boolean; + isRuleCustomizationEnabled: boolean; }): RuleSource => { if (!isKnownPrebuiltRule) { return { @@ -38,11 +40,15 @@ export const calculateRuleSourceFromAsset = ({ if (assetWithMatchingVersion == null) { return { type: 'external', - is_customized: true, + is_customized: isRuleCustomizationEnabled ? true : false, }; } - const isCustomized = calculateIsCustomized(assetWithMatchingVersion, rule); + const isCustomized = calculateIsCustomized({ + baseRule: assetWithMatchingVersion, + nextRule: rule, + isRuleCustomizationEnabled, + }); return { type: 'external', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index 39c937f4645a7..426109bbeb3bb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -138,6 +138,7 @@ describe('ruleSourceImporter', () => { rule, prebuiltRuleAssetsByRuleId: { 'rule-1': expect.objectContaining({ rule_id: 'rule-1' }) }, isKnownPrebuiltRule: true, + isRuleCustomizationEnabled: true, }); }); @@ -166,6 +167,7 @@ describe('ruleSourceImporter', () => { rule, prebuiltRuleAssetsByRuleId: { 'rule-1': expect.objectContaining({ rule_id: 'rule-1' }) }, isKnownPrebuiltRule: true, + isRuleCustomizationEnabled: true, }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts index 1f5c2c5aa543b..0c553e6bb7c56 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -143,6 +143,8 @@ export class RuleSourceImporter implements IRuleSourceImporter { rule, prebuiltRuleAssetsByRuleId: this.matchingAssetsByRuleId, isKnownPrebuiltRule: this.availableRuleAssetIds.has(rule.rule_id), + isRuleCustomizationEnabled: + this.config.experimentalFeatures.prebuiltRulesCustomizationEnabled, }); } diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index 6b2718bac7704..c2b3782d405d0 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -152,6 +152,7 @@ export class RequestContextFactory implements IRequestContextFactory { actionsClient, savedObjectsClient: coreContext.savedObjects.client, mlAuthz, + isRuleCustomizationEnabled: config.experimentalFeatures.prebuiltRulesCustomizationEnabled, }); }), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/ess.config.ts new file mode 100644 index 0000000000000..2e1b278c9247f --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/ess.config.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile( + require.resolve('../../../../../../../config/ess/config.base.trial') + ); + + const testConfig = { + ...functionalConfig.getAll(), + testFiles: [require.resolve('..')], + junit: { + reportName: + 'Rules Management - Prebuilt Rule Customization Disabled Integration Tests - ESS Env', + }, + }; + + return testConfig; +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/serverless.config.ts new file mode 100644 index 0000000000000..462a971d8dda7 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/serverless.config.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../../../../../../../config/serverless/config.base'; + +export default createTestConfig({ + testFiles: [require.resolve('..')], + junit: { + reportName: + 'Rules Management - Prebuilt Rule Customization Disabled Integration Tests - Serverless Env', + }, + kbnTestServerArgs: [], +}); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/index.ts new file mode 100644 index 0000000000000..0d78605426dbf --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../../../ftr_provider_context'; + +export default ({ loadTestFile }: FtrProviderContext): void => { + describe('Rules Management - Prebuilt Rules - Prebuilt Rule Customization Disabled', function () { + loadTestFile(require.resolve('./is_customized_calculation')); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/is_customized_calculation.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/is_customized_calculation.ts new file mode 100644 index 0000000000000..e4ae135b481ed --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/is_customized_calculation.ts @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { + BulkActionEditTypeEnum, + BulkActionTypeEnum, +} from '@kbn/security-solution-plugin/common/api/detection_engine'; +import { deleteAllRules } from '../../../../../../../common/utils/security_solution'; +import { FtrProviderContext } from '../../../../../../ftr_provider_context'; +import { + deleteAllPrebuiltRuleAssets, + createRuleAssetSavedObject, + createPrebuiltRuleAssetSavedObjects, + installPrebuiltRules, +} from '../../../../utils'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const securitySolutionApi = getService('securitySolutionApi'); + const log = getService('log'); + const es = getService('es'); + + const ruleAsset = createRuleAssetSavedObject({ + rule_id: 'test-rule-id', + }); + + describe('@ess @serverless @skipInServerlessMKI is_customized calculation with disabled customization', () => { + beforeEach(async () => { + await deleteAllRules(supertest, log); + await deleteAllPrebuiltRuleAssets(es, log); + }); + + it('should set is_customized to "false" on prebuilt rule PATCH', async () => { + await createPrebuiltRuleAssetSavedObjects(es, [ruleAsset]); + await installPrebuiltRules(es, supertest); + + const { body: findResult } = await securitySolutionApi + .findRules({ + query: { + per_page: 1, + filter: `alert.attributes.params.immutable: true`, + }, + }) + .expect(200); + const prebuiltRule = findResult.data[0]; + + // Check that the rule has been created and is not customized + expect(prebuiltRule).not.toBeNull(); + expect(prebuiltRule.rule_source.is_customized).toEqual(false); + + const { body } = await securitySolutionApi + .patchRule({ + body: { + rule_id: 'test-rule-id', + name: 'some other rule name', + }, + }) + .expect(200); + + // Check that the rule name has been updated and the rule is still not customized + expect(body).toEqual( + expect.objectContaining({ + name: 'some other rule name', + }) + ); + expect(body.rule_source.is_customized).toEqual(false); + }); + + it('should set is_customized to "false" on prebuilt rule UPDATE', async () => { + await createPrebuiltRuleAssetSavedObjects(es, [ruleAsset]); + await installPrebuiltRules(es, supertest); + + const { body: findResult } = await securitySolutionApi + .findRules({ + query: { + per_page: 1, + filter: `alert.attributes.params.immutable: true`, + }, + }) + .expect(200); + const prebuiltRule = findResult.data[0]; + + // Check that the rule has been created and is not customized + expect(prebuiltRule).not.toBeNull(); + expect(prebuiltRule.rule_source.is_customized).toEqual(false); + + const { body } = await securitySolutionApi + .updateRule({ + body: { + ...prebuiltRule, + id: undefined, // id together with rule_id is not allowed + name: 'some other rule name', + }, + }) + .expect(200); + + // Check that the rule name has been updated and the rule is still not customized + expect(body).toEqual( + expect.objectContaining({ + name: 'some other rule name', + }) + ); + expect(body.rule_source.is_customized).toEqual(false); + }); + + it('should not allow prebuilt rule customization on import', async () => { + await createPrebuiltRuleAssetSavedObjects(es, [ruleAsset]); + await installPrebuiltRules(es, supertest); + + const { body: findResult } = await securitySolutionApi + .findRules({ + query: { + per_page: 1, + filter: `alert.attributes.params.immutable: true`, + }, + }) + .expect(200); + const prebuiltRule = findResult.data[0]; + + // Check that the rule has been created and is not customized + expect(prebuiltRule).not.toBeNull(); + expect(prebuiltRule.rule_source.is_customized).toEqual(false); + + const ruleBuffer = Buffer.from( + JSON.stringify({ + ...prebuiltRule, + name: 'some other rule name', + }) + ); + + const { body } = await securitySolutionApi + .importRules({ query: {} }) + .attach('file', ruleBuffer, 'rules.ndjson') + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(200); + + expect(body).toMatchObject({ + rules_count: 1, + success: false, + success_count: 0, + errors: [ + { + error: { + message: expect.stringContaining('Importing prebuilt rules is not supported'), + }, + rule_id: 'test-rule-id', + }, + ], + }); + + // Check that the rule has not been customized + const { body: importedRule } = await securitySolutionApi.readRule({ + query: { rule_id: prebuiltRule.rule_id }, + }); + expect(importedRule.rule_source.is_customized).toEqual(false); + }); + + it('should not allow rule customization on bulk edit', async () => { + await createPrebuiltRuleAssetSavedObjects(es, [ruleAsset]); + await installPrebuiltRules(es, supertest); + + const { body: findResult } = await securitySolutionApi + .findRules({ + query: { + per_page: 1, + filter: `alert.attributes.params.immutable: true`, + }, + }) + .expect(200); + const prebuiltRule = findResult.data[0]; + + // Check that the rule has been created and is not customized + expect(prebuiltRule).not.toBeNull(); + expect(prebuiltRule.rule_source.is_customized).toEqual(false); + + const { body: bulkResult } = await securitySolutionApi + .performRulesBulkAction({ + query: {}, + body: { + ids: [prebuiltRule.id], + action: BulkActionTypeEnum.edit, + [BulkActionTypeEnum.edit]: [ + { + type: BulkActionEditTypeEnum.add_tags, + value: ['test'], + }, + ], + }, + }) + .expect(500); + + expect(bulkResult).toMatchObject( + expect.objectContaining({ + attributes: expect.objectContaining({ + summary: { + failed: 1, + skipped: 0, + succeeded: 0, + total: 1, + }, + errors: [expect.objectContaining({ message: "Elastic rule can't be edited" })], + }), + }) + ); + + // Check that the rule has not been customized + const { body: ruleAfterUpdate } = await securitySolutionApi.readRule({ + query: { rule_id: prebuiltRule.rule_id }, + }); + expect(ruleAfterUpdate.rule_source.is_customized).toEqual(false); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/configs/ess.config.ts similarity index 91% rename from x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/ess.config.ts rename to x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/configs/ess.config.ts index eee14323c9b98..ff46ebb70d7c8 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/ess.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/configs/ess.config.ts @@ -17,7 +17,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { testFiles: [require.resolve('..')], junit: { reportName: - 'Rules Management - Prebuilt Rule Customization Integration Tests - ESS Env - Trial License', + 'Rules Management - Prebuilt Rule Customization Enabled Integration Tests - ESS Env', }, }; testConfig.kbnTestServer.serverArgs = testConfig.kbnTestServer.serverArgs.map((arg: string) => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/configs/serverless.config.ts similarity index 84% rename from x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/serverless.config.ts rename to x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/configs/serverless.config.ts index 79a23c85d2279..8c1d777665667 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/configs/serverless.config.ts @@ -11,7 +11,7 @@ export default createTestConfig({ testFiles: [require.resolve('..')], junit: { reportName: - 'Rules Management - Prebuilt Rule Customization Integration Tests - Serverless Env - Complete Tier', + 'Rules Management - Prebuilt Rule Customization Enabled Integration Tests - Serverless Env', }, kbnTestServerArgs: [ `--xpack.securitySolution.enableExperimental=${JSON.stringify([ diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/import_rules.ts similarity index 100% rename from x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts rename to x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/import_rules.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/index.ts similarity index 94% rename from x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts rename to x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/index.ts index 58904243e51ca..d89f8ed5d49d3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/index.ts @@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../../../../../ftr_provider_context'; export default ({ loadTestFile }: FtrProviderContext): void => { - describe('Rules Management - Prebuilt Rules - Prebuilt Rule Customization', function () { + describe('Rules Management - Prebuilt Rules - Prebuilt Rule Customization Enabled', function () { loadTestFile(require.resolve('./is_customized_calculation')); loadTestFile(require.resolve('./import_rules')); loadTestFile(require.resolve('./rules_export')); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/is_customized_calculation.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/is_customized_calculation.ts similarity index 100% rename from x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/is_customized_calculation.ts rename to x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/is_customized_calculation.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/rules_export.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/rules_export.ts similarity index 100% rename from x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/rules_export.ts rename to x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/rules_export.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/basic_license_essentials_tier/patch_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/basic_license_essentials_tier/patch_rules.ts index 86dde0735424e..a47f84ef930ad 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/basic_license_essentials_tier/patch_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/basic_license_essentials_tier/patch_rules.ts @@ -7,25 +7,21 @@ import expect from 'expect'; +import { createRule, deleteAllRules } from '../../../../../../common/utils/security_solution'; import { FtrProviderContext } from '../../../../../ftr_provider_context'; import { + createHistoricalPrebuiltRuleAssetSavedObjects, + createRuleAssetSavedObject, + deleteAllPrebuiltRuleAssets, + getCustomQueryRuleParams, getSimpleRule, getSimpleRuleOutput, - getCustomQueryRuleParams, + getSimpleRuleOutputWithoutRuleId, + installPrebuiltRules, removeServerGeneratedProperties, removeServerGeneratedPropertiesIncludingRuleId, - getSimpleRuleOutputWithoutRuleId, updateUsername, - createHistoricalPrebuiltRuleAssetSavedObjects, - installPrebuiltRules, - createRuleAssetSavedObject, } from '../../../utils'; -import { - createAlertsIndex, - deleteAllRules, - createRule, - deleteAllAlerts, -} from '../../../../../../common/utils/security_solution'; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); @@ -37,12 +33,8 @@ export default ({ getService }: FtrProviderContext) => { describe('@ess @serverless @serverlessQA patch_rules', () => { describe('patch rules', () => { beforeEach(async () => { - await createAlertsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteAllAlerts(supertest, log, es); await deleteAllRules(supertest, log); + await deleteAllPrebuiltRuleAssets(es, log); }); it('should patch a single rule property of name using a rule_id', async () => { @@ -262,10 +254,6 @@ export default ({ getService }: FtrProviderContext) => { }); describe('max signals', () => { - afterEach(async () => { - await deleteAllRules(supertest, log); - }); - it('does NOT patch a rule when max_signals is less than 1', async () => { await securitySolutionApi.createRule({ body: getCustomQueryRuleParams({ rule_id: 'rule-1', max_signals: 100 }), From bcd442239f0eaa6ba84579ea19311ad29221c01f Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Tue, 3 Dec 2024 13:25:30 +0100 Subject: [PATCH 02/51] [EDR Workflows] Fix wrong endpoint link (#202434) --- .../endpoint_hosts/view/hooks/use_endpoint_action_items.tsx | 2 +- .../public/management/pages/endpoint_hosts/view/index.test.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/hooks/use_endpoint_action_items.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/hooks/use_endpoint_action_items.tsx index bb6102e8051c4..579561573488c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/hooks/use_endpoint_action_items.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/hooks/use_endpoint_action_items.tsx @@ -56,7 +56,7 @@ export const useEndpointActionItems = ( const endpointMetadata = endpointInfo.metadata; const isIsolated = isEndpointHostIsolated(endpointMetadata); const endpointId = endpointMetadata.agent.id; - const endpointHostName = endpointMetadata.host.hostname; + const endpointHostName = endpointMetadata.host.hostname.toLowerCase(); const fleetAgentId = endpointMetadata.elastic.agent.id; const { show, selected_endpoint: _selectedEndpoint, ...currentUrlParams } = allCurrentUrlParams; const endpointActionsPath = getEndpointDetailsPath({ diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx index d0b05404dc16d..57b0dde41177d 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx @@ -1306,7 +1306,7 @@ describe('when on the endpoint list page', () => { it('navigates to the Security Solution Host Details page', async () => { const hostLink = await renderResult.findByTestId('hostLink'); expect(hostLink.getAttribute('href')).toEqual( - `${APP_PATH}/hosts/${hostInfo[0].metadata.host.hostname}` + `${APP_PATH}/hosts/${hostInfo[0].metadata.host.hostname.toLowerCase()}` ); }); it('navigates to the correct Ingest Agent Policy page', async () => { From d1e5aa158248fef3a86a59be4212f0a8d511ff91 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 3 Dec 2024 23:44:43 +1100 Subject: [PATCH 03/51] [Console] Update console definitions (#202394) This PR updates the console definitions to match the latest ones from the @elastic/elasticsearch-specification repo. --- .../json/generated/async_search.status.json | 7 ++++++- .../json/generated/async_search.submit.json | 4 ---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/async_search.status.json b/src/plugins/console/server/lib/spec_definitions/json/generated/async_search.status.json index 66d774b605076..4e605ca1d0efe 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/async_search.status.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/async_search.status.json @@ -4,7 +4,12 @@ "error_trace": "__flag__", "filter_path": [], "human": "__flag__", - "pretty": "__flag__" + "pretty": "__flag__", + "keep_alive": [ + "5d", + "-1", + "0" + ] }, "methods": [ "GET" diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/async_search.submit.json b/src/plugins/console/server/lib/spec_definitions/json/generated/async_search.submit.json index fcc429811827e..3da69d695ef76 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/async_search.submit.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/async_search.submit.json @@ -48,10 +48,6 @@ ], "request_cache": "__flag__", "routing": "", - "scroll": [ - "-1", - "0" - ], "search_type": [ "query_then_fetch", "dfs_query_then_fetch" From a75a92b284a860ecbb5e3accef97d3044f5ac4a4 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 3 Dec 2024 14:35:34 +0100 Subject: [PATCH 04/51] [Dataset Quality][Observability Onboarding] Update axios headers (#202412) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 📓 Summary To ensure Kibana features/tests don't break after the update to v9.0.0, we need to provide the `'x-elastic-internal-origin': 'kibana'` header is set on API providers that are not the internal core HTTP service. These changes fix a couple of utilities using the axios library for testing purpose. --------- Co-authored-by: Marco Antonio Ghiani --- .../create_dataset_quality_users/helpers/call_kibana.ts | 8 ++++++-- .../helpers/call_kibana.ts | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/test_helpers/create_dataset_quality_users/helpers/call_kibana.ts b/x-pack/plugins/observability_solution/dataset_quality/server/test_helpers/create_dataset_quality_users/helpers/call_kibana.ts index 879b02f8a93c5..5f36a8a4204f2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/test_helpers/create_dataset_quality_users/helpers/call_kibana.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/test_helpers/create_dataset_quality_users/helpers/call_kibana.ts @@ -24,14 +24,18 @@ export async function callKibana({ ...options, baseURL: baseUrl, auth: { username, password }, - headers: { 'kbn-xsrf': 'true', ...options.headers }, + headers: { 'kbn-xsrf': 'true', 'x-elastic-internal-origin': 'kibana', ...options.headers }, }); return data; } const getBaseUrl = once(async (kibanaHostname: string) => { try { - await axios.request({ url: kibanaHostname, maxRedirects: 0 }); + await axios.request({ + url: kibanaHostname, + maxRedirects: 0, + headers: { 'x-elastic-internal-origin': 'kibana' }, + }); } catch (e) { if (isAxiosError(e)) { const location = e.response?.headers?.location ?? ''; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/test_helpers/create_observability_onboarding_users/helpers/call_kibana.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/test_helpers/create_observability_onboarding_users/helpers/call_kibana.ts index 879b02f8a93c5..5f36a8a4204f2 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/test_helpers/create_observability_onboarding_users/helpers/call_kibana.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/test_helpers/create_observability_onboarding_users/helpers/call_kibana.ts @@ -24,14 +24,18 @@ export async function callKibana({ ...options, baseURL: baseUrl, auth: { username, password }, - headers: { 'kbn-xsrf': 'true', ...options.headers }, + headers: { 'kbn-xsrf': 'true', 'x-elastic-internal-origin': 'kibana', ...options.headers }, }); return data; } const getBaseUrl = once(async (kibanaHostname: string) => { try { - await axios.request({ url: kibanaHostname, maxRedirects: 0 }); + await axios.request({ + url: kibanaHostname, + maxRedirects: 0, + headers: { 'x-elastic-internal-origin': 'kibana' }, + }); } catch (e) { if (isAxiosError(e)) { const location = e.response?.headers?.location ?? ''; From 5b391b0abe5158249ae83a393b3fdf8fdcba3c6b Mon Sep 17 00:00:00 2001 From: Kevin Delemme Date: Tue, 3 Dec 2024 08:44:15 -0500 Subject: [PATCH 05/51] fix(slo): Overview Embeddable drilldown actions (#201870) --- .../plugins/embeddable_enhanced/kibana.jsonc | 2 +- .../observability_solution/slo/kibana.jsonc | 5 +- .../slo/overview/slo_embeddable_factory.tsx | 385 +++++++++--------- .../public/embeddable/slo/overview/types.ts | 17 +- .../slo/public/types.ts | 2 + .../observability_solution/slo/tsconfig.json | 1 + .../stats_overview_embeddable_factory.tsx | 1 + 7 files changed, 216 insertions(+), 197 deletions(-) diff --git a/x-pack/plugins/embeddable_enhanced/kibana.jsonc b/x-pack/plugins/embeddable_enhanced/kibana.jsonc index d795afa4d7938..3b9632d4bf36c 100644 --- a/x-pack/plugins/embeddable_enhanced/kibana.jsonc +++ b/x-pack/plugins/embeddable_enhanced/kibana.jsonc @@ -5,7 +5,7 @@ "@elastic/kibana-presentation" ], "group": "platform", - "visibility": "private", + "visibility": "shared", "description": "Extends embeddable plugin with more functionality", "plugin": { "id": "embeddableEnhanced", diff --git a/x-pack/plugins/observability_solution/slo/kibana.jsonc b/x-pack/plugins/observability_solution/slo/kibana.jsonc index 11eca10c6c8db..1a90f45c8cd9b 100644 --- a/x-pack/plugins/observability_solution/slo/kibana.jsonc +++ b/x-pack/plugins/observability_solution/slo/kibana.jsonc @@ -44,10 +44,11 @@ "optionalPlugins": [ "cloud", "discover", + "embeddableEnhanced", "observabilityAIAssistant", + "security", "serverless", - "spaces", - "security" + "spaces" ], "requiredBundles": [ "controls", diff --git a/x-pack/plugins/observability_solution/slo/public/embeddable/slo/overview/slo_embeddable_factory.tsx b/x-pack/plugins/observability_solution/slo/public/embeddable/slo/overview/slo_embeddable_factory.tsx index 57de174194976..c1b19c7381acb 100644 --- a/x-pack/plugins/observability_solution/slo/public/embeddable/slo/overview/slo_embeddable_factory.tsx +++ b/x-pack/plugins/observability_solution/slo/public/embeddable/slo/overview/slo_embeddable_factory.tsx @@ -14,6 +14,7 @@ import { EuiThemeProvider } from '@kbn/kibana-react-plugin/common'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { fetch$, + getUnchangingComparator, initializeTitles, useBatchedPublishingSubjects, } from '@kbn/presentation-publishing'; @@ -43,200 +44,210 @@ export const getOverviewEmbeddableFactory = ({ coreStart: CoreStart; pluginsStart: SLOPublicPluginsStart; sloClient: SLORepositoryClient; -}) => { - const factory: ReactEmbeddableFactory< - SloOverviewEmbeddableState, - SloOverviewEmbeddableState, - SloOverviewApi - > = { - type: SLO_OVERVIEW_EMBEDDABLE_ID, - deserializeState: (state) => { - return state.rawState as SloOverviewEmbeddableState; - }, - buildEmbeddable: async (state, buildApi, uuid, parentApi) => { - const deps = { ...coreStart, ...pluginsStart }; - async function onEdit() { - try { - const { openSloConfiguration } = await import('./slo_overview_open_configuration'); +}): ReactEmbeddableFactory< + SloOverviewEmbeddableState, + SloOverviewEmbeddableState, + SloOverviewApi +> => ({ + type: SLO_OVERVIEW_EMBEDDABLE_ID, + deserializeState: (state) => { + return state.rawState as SloOverviewEmbeddableState; + }, + buildEmbeddable: async (state, buildApi, uuid, parentApi) => { + const deps = { ...coreStart, ...pluginsStart }; - const result = await openSloConfiguration( - coreStart, - pluginsStart, - sloClient, - api.getSloGroupOverviewConfig() - ); - api.updateSloGroupOverviewConfig(result as GroupSloCustomInput); - } catch (e) { - return Promise.reject(); - } - } - const { titlesApi, titleComparators, serializeTitles } = initializeTitles(state); - const defaultTitle$ = new BehaviorSubject(getOverviewPanelTitle()); - const sloId$ = new BehaviorSubject(state.sloId); - const sloInstanceId$ = new BehaviorSubject(state.sloInstanceId); - const showAllGroupByInstances$ = new BehaviorSubject(state.showAllGroupByInstances); - const overviewMode$ = new BehaviorSubject(state.overviewMode); - const groupFilters$ = new BehaviorSubject(state.groupFilters); - const remoteName$ = new BehaviorSubject(state.remoteName); - const reload$ = new Subject(); + const dynamicActionsApi = deps.embeddableEnhanced?.initializeReactEmbeddableDynamicActions( + uuid, + () => titlesApi.panelTitle.getValue(), + state + ); - const api = buildApi( - { - ...titlesApi, - defaultPanelTitle: defaultTitle$, - getTypeDisplayName: () => - i18n.translate('xpack.slo.editSloOverviewEmbeddableTitle.typeDisplayName', { - defaultMessage: 'criteria', - }), - isEditingEnabled: () => api.getSloGroupOverviewConfig().overviewMode === 'groups', - onEdit: async () => { - onEdit(); - }, - serializeState: () => { - return { - rawState: { - ...serializeTitles(), - sloId: sloId$.getValue(), - sloInstanceId: sloInstanceId$.getValue(), - showAllGroupByInstances: showAllGroupByInstances$.getValue(), - overviewMode: overviewMode$.getValue(), - groupFilters: groupFilters$.getValue(), - remoteName: remoteName$.getValue(), - }, - }; - }, - getSloGroupOverviewConfig: () => { - return { - groupFilters: groupFilters$.getValue(), + const maybeStopDynamicActions = dynamicActionsApi?.startDynamicActions(); + + const { titlesApi, titleComparators, serializeTitles } = initializeTitles(state); + const defaultTitle$ = new BehaviorSubject(getOverviewPanelTitle()); + const sloId$ = new BehaviorSubject(state.sloId); + const sloInstanceId$ = new BehaviorSubject(state.sloInstanceId); + const showAllGroupByInstances$ = new BehaviorSubject(state.showAllGroupByInstances); + const overviewMode$ = new BehaviorSubject(state.overviewMode); + const groupFilters$ = new BehaviorSubject(state.groupFilters); + const remoteName$ = new BehaviorSubject(state.remoteName); + const reload$ = new Subject(); + + const api = buildApi( + { + ...titlesApi, + ...(dynamicActionsApi?.dynamicActionsApi ?? {}), + supportedTriggers: () => [], + defaultPanelTitle: defaultTitle$, + getTypeDisplayName: () => + i18n.translate('xpack.slo.editSloOverviewEmbeddableTitle.typeDisplayName', { + defaultMessage: 'criteria', + }), + isEditingEnabled: () => api.getSloGroupOverviewConfig().overviewMode === 'groups', + onEdit: async function onEdit() { + try { + const { openSloConfiguration } = await import('./slo_overview_open_configuration'); + + const result = await openSloConfiguration( + coreStart, + pluginsStart, + sloClient, + api.getSloGroupOverviewConfig() + ); + api.updateSloGroupOverviewConfig(result as GroupSloCustomInput); + } catch (e) { + return Promise.reject(); + } + }, + serializeState: () => { + return { + rawState: { + ...serializeTitles(), + sloId: sloId$.getValue(), + sloInstanceId: sloInstanceId$.getValue(), + showAllGroupByInstances: showAllGroupByInstances$.getValue(), overviewMode: overviewMode$.getValue(), - }; - }, - updateSloGroupOverviewConfig: (update: GroupSloCustomInput) => { - groupFilters$.next(update.groupFilters); - }, + groupFilters: groupFilters$.getValue(), + remoteName: remoteName$.getValue(), + ...(dynamicActionsApi?.serializeDynamicActions?.() ?? {}), + }, + }; }, - { - sloId: [sloId$, (value) => sloId$.next(value)], - sloInstanceId: [sloInstanceId$, (value) => sloInstanceId$.next(value)], - groupFilters: [groupFilters$, (value) => groupFilters$.next(value)], - showAllGroupByInstances: [ - showAllGroupByInstances$, - (value) => showAllGroupByInstances$.next(value), - ], - remoteName: [remoteName$, (value) => remoteName$.next(value)], - overviewMode: [overviewMode$, (value) => overviewMode$.next(value)], - ...titleComparators, - } - ); - - const fetchSubscription = fetch$(api) - .pipe() - .subscribe((next) => { - reload$.next(next.isReload); - }); + getSloGroupOverviewConfig: () => { + return { + groupFilters: groupFilters$.getValue(), + overviewMode: overviewMode$.getValue(), + }; + }, + updateSloGroupOverviewConfig: (update: GroupSloCustomInput) => { + groupFilters$.next(update.groupFilters); + }, + }, + { + sloId: [sloId$, (value) => sloId$.next(value)], + sloInstanceId: [sloInstanceId$, (value) => sloInstanceId$.next(value)], + groupFilters: [groupFilters$, (value) => groupFilters$.next(value)], + showAllGroupByInstances: [ + showAllGroupByInstances$, + (value) => showAllGroupByInstances$.next(value), + ], + remoteName: [remoteName$, (value) => remoteName$.next(value)], + overviewMode: [overviewMode$, (value) => overviewMode$.next(value)], + ...titleComparators, + ...(dynamicActionsApi?.dynamicActionsComparator ?? { + enhancements: getUnchangingComparator(), + }), + } + ); - return { - api, - Component: () => { - const [ - sloId, - sloInstanceId, - showAllGroupByInstances, - overviewMode, - groupFilters, - remoteName, - ] = useBatchedPublishingSubjects( - sloId$, - sloInstanceId$, - showAllGroupByInstances$, - overviewMode$, - groupFilters$, - remoteName$ - ); + const fetchSubscription = fetch$(api) + .pipe() + .subscribe((next) => { + reload$.next(next.isReload); + }); - useEffect(() => { - return () => { - fetchSubscription.unsubscribe(); - }; - }, []); - const renderOverview = () => { - if (overviewMode === 'groups') { - const groupBy = groupFilters?.groupBy ?? 'status'; - const kqlQuery = groupFilters?.kqlQuery ?? ''; - const groups = groupFilters?.groups ?? []; - return ( -
css` - width: 100%; - padding: ${euiTheme.size.xs} ${euiTheme.size.base}; - overflow: scroll; + return { + api, + Component: () => { + const [ + sloId, + sloInstanceId, + showAllGroupByInstances, + overviewMode, + groupFilters, + remoteName, + ] = useBatchedPublishingSubjects( + sloId$, + sloInstanceId$, + showAllGroupByInstances$, + overviewMode$, + groupFilters$, + remoteName$ + ); - .euiAccordion__buttonContent { - min-width: ${euiTheme.base * 6}px; - } - `} - > - - css` - margin-top: ${euiTheme.base * 1.25}px; - `} - > - - - -
- ); - } else { - return ( - - ); - } + useEffect(() => { + return () => { + fetchSubscription.unsubscribe(); + maybeStopDynamicActions?.stopDynamicActions(); }; + }, []); + const renderOverview = () => { + if (overviewMode === 'groups') { + const groupBy = groupFilters?.groupBy ?? 'status'; + const kqlQuery = groupFilters?.kqlQuery ?? ''; + const groups = groupFilters?.groups ?? []; + return ( +
css` + width: 100%; + padding: ${euiTheme.size.xs} ${euiTheme.size.base}; + overflow: scroll; - const queryClient = new QueryClient(); - - return ( - - - - + + css` + margin-top: ${euiTheme.base * 1.25}px; + `} > - - {showAllGroupByInstances ? ( - - ) : ( - renderOverview() - )} - - - - - - ); - }, - }; - }, - }; - return factory; -}; + + + +
+ ); + } else { + return ( + + ); + } + }; + + const queryClient = new QueryClient(); + + return ( + + + + + + {showAllGroupByInstances ? ( + + ) : ( + renderOverview() + )} + + + + + + ); + }, + }; + }, +}); diff --git a/x-pack/plugins/observability_solution/slo/public/embeddable/slo/overview/types.ts b/x-pack/plugins/observability_solution/slo/public/embeddable/slo/overview/types.ts index 3c2866077aaa6..d79a0ecd8a4dc 100644 --- a/x-pack/plugins/observability_solution/slo/public/embeddable/slo/overview/types.ts +++ b/x-pack/plugins/observability_solution/slo/public/embeddable/slo/overview/types.ts @@ -4,15 +4,16 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import type { DynamicActionsSerializedState } from '@kbn/embeddable-enhanced-plugin/public/plugin'; +import { DefaultEmbeddableApi } from '@kbn/embeddable-plugin/public'; +import { Filter } from '@kbn/es-query'; +import type { EmbeddableApiContext, HasSupportedTriggers } from '@kbn/presentation-publishing'; import { - SerializedTitles, - PublishesWritablePanelTitle, - PublishesPanelTitle, HasEditCapabilities, + PublishesPanelTitle, + PublishesWritablePanelTitle, + SerializedTitles, } from '@kbn/presentation-publishing'; -import type { EmbeddableApiContext } from '@kbn/presentation-publishing'; -import { DefaultEmbeddableApi } from '@kbn/embeddable-plugin/public'; -import { Filter } from '@kbn/es-query'; export type OverviewMode = 'single' | 'groups'; export type GroupBy = 'slo.tags' | 'status' | 'slo.indicator.type'; @@ -39,6 +40,7 @@ export type GroupSloCustomInput = SloConfigurationProps & { }; export type SloOverviewEmbeddableState = SerializedTitles & + Partial & Partial & Partial; @@ -46,7 +48,8 @@ export type SloOverviewApi = DefaultEmbeddableApi & PublishesWritablePanelTitle & PublishesPanelTitle & HasSloGroupOverviewConfig & - HasEditCapabilities; + HasEditCapabilities & + HasSupportedTriggers; export interface HasSloGroupOverviewConfig { getSloGroupOverviewConfig: () => GroupSloCustomInput; diff --git a/x-pack/plugins/observability_solution/slo/public/types.ts b/x-pack/plugins/observability_solution/slo/public/types.ts index 964fb9b289ea3..80439c4a4134c 100644 --- a/x-pack/plugins/observability_solution/slo/public/types.ts +++ b/x-pack/plugins/observability_solution/slo/public/types.ts @@ -15,6 +15,7 @@ import { DataViewFieldEditorStart } from '@kbn/data-view-field-editor-plugin/pub import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import type { DiscoverStart } from '@kbn/discover-plugin/public'; import type { DiscoverSharedPublicStart } from '@kbn/discover-shared-plugin/public'; +import type { EmbeddableEnhancedPluginStart } from '@kbn/embeddable-enhanced-plugin/public'; import type { EmbeddableSetup, EmbeddableStart } from '@kbn/embeddable-plugin/public'; import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import type { LensPublicStart } from '@kbn/lens-plugin/public'; @@ -82,6 +83,7 @@ export interface SLOPublicPluginsStart { discover?: DiscoverStart; discoverShared: DiscoverSharedPublicStart; embeddable: EmbeddableStart; + embeddableEnhanced?: EmbeddableEnhancedPluginStart; fieldFormats: FieldFormatsStart; lens: LensPublicStart; licensing: LicensingPluginStart; diff --git a/x-pack/plugins/observability_solution/slo/tsconfig.json b/x-pack/plugins/observability_solution/slo/tsconfig.json index 3b01431c3fb45..125a8463be595 100644 --- a/x-pack/plugins/observability_solution/slo/tsconfig.json +++ b/x-pack/plugins/observability_solution/slo/tsconfig.json @@ -30,6 +30,7 @@ "@kbn/alerting-plugin", "@kbn/rison", "@kbn/embeddable-plugin", + "@kbn/embeddable-enhanced-plugin", "@kbn/lens-plugin", "@kbn/ui-theme", "@kbn/es-query", diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/embeddables/stats_overview/stats_overview_embeddable_factory.tsx b/x-pack/plugins/observability_solution/synthetics/public/apps/embeddables/stats_overview/stats_overview_embeddable_factory.tsx index 83b37f080d422..df311819399f3 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/embeddables/stats_overview/stats_overview_embeddable_factory.tsx +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/embeddables/stats_overview/stats_overview_embeddable_factory.tsx @@ -67,6 +67,7 @@ export const getStatsOverviewEmbeddableFactory = ( i18n.translate('xpack.synthetics.editSloOverviewEmbeddableTitle.typeDisplayName', { defaultMessage: 'filters', }), + isEditingEnabled: () => true, onEdit: async () => { try { From b217f1acbdce4d9c0288c87e9afa470038cf6557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 3 Dec 2024 14:48:19 +0100 Subject: [PATCH 06/51] [Obs AI Assistant] Perform index creation at startup (#201362) Currently the knowledge base creates index assets (index templates, index components) lazily when the user interacts with the assistant. This prevents running the semantic text migrations (added in https://github.com/elastic/kibana/pull/186499) when Kibana starts because the mappings have not yet been updated. Additionally, this PR also increases `min_number_of_allocations` to 1 to ensure at least one ML node is available at all times. --- .../server/plugin.ts | 5 +- .../server/service/client/index.test.ts | 8 +- .../server/service/client/index.ts | 32 +++- .../server/service/index.ts | 156 ++++-------------- .../server/service/inference_endpoint.ts | 97 ++++++++--- .../server/service/kb_component_template.ts | 4 - .../service/knowledge_base_service/index.ts | 59 +------ .../setup_conversation_and_kb_index_assets.ts | 108 ++++++++++++ ...ter_migrate_knowledge_base_entries_task.ts | 99 +++++++---- 9 files changed, 316 insertions(+), 252 deletions(-) create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/service/setup_conversation_and_kb_index_assets.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts index 7949276ac6aba..b8de2f5688373 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts @@ -134,8 +134,9 @@ export class ObservabilityAIAssistantPlugin core, taskManager: plugins.taskManager, logger: this.logger, - }).catch((error) => { - this.logger.error(`Failed to register migrate knowledge base entries task: ${error}`); + config: this.config, + }).catch((e) => { + this.logger.error(`Knowledge base migration was not successfully: ${e.message}`); }); service.register(registerFunctions); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts index 8da2a0d843b11..f6aa0dfab2726 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts @@ -5,7 +5,7 @@ * 2.0. */ import type { ActionsClient } from '@kbn/actions-plugin/server/actions_client'; -import type { ElasticsearchClient, IUiSettingsClient, Logger } from '@kbn/core/server'; +import type { CoreSetup, ElasticsearchClient, IUiSettingsClient, Logger } from '@kbn/core/server'; import type { DeeplyMockedKeys } from '@kbn/utility-types-jest'; import { waitFor } from '@testing-library/react'; import { last, merge, repeat } from 'lodash'; @@ -27,7 +27,9 @@ import { CONTEXT_FUNCTION_NAME } from '../../functions/context'; import { ChatFunctionClient } from '../chat_function_client'; import type { KnowledgeBaseService } from '../knowledge_base_service'; import { observableIntoStream } from '../util/observable_into_stream'; -import { CreateChatCompletionResponseChunk } from './adapters/process_openai_stream'; +import type { CreateChatCompletionResponseChunk } from './adapters/process_openai_stream'; +import type { ObservabilityAIAssistantConfig } from '../../config'; +import type { ObservabilityAIAssistantPluginStartDependencies } from '../../types'; type ChunkDelta = CreateChatCompletionResponseChunk['choices'][number]['delta']; @@ -177,6 +179,8 @@ describe('Observability AI Assistant client', () => { functionClientMock.getAdhocInstructions.mockReturnValue([]); return new ObservabilityAIAssistantClient({ + config: {} as ObservabilityAIAssistantConfig, + core: {} as CoreSetup, actionsClient: actionsClientMock, uiSettingsClient: uiSettingsClientMock, esClient: { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 2bd2fdcf22462..107bed3cac7be 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -7,7 +7,7 @@ import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { notFound } from '@hapi/boom'; import type { ActionsClient } from '@kbn/actions-plugin/server'; -import type { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; +import type { CoreSetup, ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { SpanKind, context } from '@opentelemetry/api'; @@ -80,13 +80,20 @@ import { LangtraceServiceProvider, withLangtraceChatCompleteSpan, } from './operators/with_langtrace_chat_complete_span'; -import { runSemanticTextKnowledgeBaseMigration } from '../task_manager_definitions/register_migrate_knowledge_base_entries_task'; +import { + runSemanticTextKnowledgeBaseMigration, + scheduleSemanticTextMigration, +} from '../task_manager_definitions/register_migrate_knowledge_base_entries_task'; +import { ObservabilityAIAssistantPluginStartDependencies } from '../../types'; +import { ObservabilityAIAssistantConfig } from '../../config'; const MAX_FUNCTION_CALLS = 8; export class ObservabilityAIAssistantClient { constructor( private readonly dependencies: { + config: ObservabilityAIAssistantConfig; + core: CoreSetup; actionsClient: PublicMethodsOf; uiSettingsClient: IUiSettingsClient; namespace: string; @@ -725,9 +732,23 @@ export class ObservabilityAIAssistantClient { return this.dependencies.knowledgeBaseService.getStatus(); }; - setupKnowledgeBase = (modelId: string | undefined) => { - const { esClient } = this.dependencies; - return this.dependencies.knowledgeBaseService.setup(esClient, modelId); + setupKnowledgeBase = async (modelId: string | undefined) => { + const { esClient, core, logger, knowledgeBaseService } = this.dependencies; + + // setup the knowledge base + const res = await knowledgeBaseService.setup(esClient, modelId); + + core + .getStartServices() + .then(([_, pluginsStart]) => { + logger.debug('Schedule semantic text migration task'); + return scheduleSemanticTextMigration(pluginsStart); + }) + .catch((error) => { + logger.error(`Failed to run semantic text migration task: ${error}`); + }); + + return res; }; resetKnowledgeBase = () => { @@ -739,6 +760,7 @@ export class ObservabilityAIAssistantClient { return runSemanticTextKnowledgeBaseMigration({ esClient: this.dependencies.esClient, logger: this.dependencies.logger, + config: this.dependencies.config, }); }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts index 9c26bebdd8388..d98799fcb63a7 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts @@ -5,22 +5,19 @@ * 2.0. */ -import type { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server/plugin'; -import { createConcreteWriteIndex, getDataStreamAdapter } from '@kbn/alerting-plugin/server'; -import type { CoreSetup, CoreStart, KibanaRequest, Logger } from '@kbn/core/server'; -import type { SecurityPluginStart } from '@kbn/security-plugin/server'; +import type { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; import { getSpaceIdFromPath } from '@kbn/spaces-plugin/common'; -import { once } from 'lodash'; import type { AssistantScope } from '@kbn/ai-assistant-common'; +import { once } from 'lodash'; +import pRetry from 'p-retry'; import { ObservabilityAIAssistantScreenContextRequest } from '../../common/types'; import type { ObservabilityAIAssistantPluginStartDependencies } from '../types'; import { ChatFunctionClient } from './chat_function_client'; import { ObservabilityAIAssistantClient } from './client'; -import { conversationComponentTemplate } from './conversation_component_template'; -import { kbComponentTemplate } from './kb_component_template'; import { KnowledgeBaseService } from './knowledge_base_service'; import type { RegistrationCallback, RespondFunctionResources } from './types'; import { ObservabilityAIAssistantConfig } from '../config'; +import { setupConversationAndKbIndexAssets } from './setup_conversation_and_kb_index_assets'; function getResourceName(resource: string) { return `.kibana-observability-ai-assistant-${resource}`; @@ -45,12 +42,15 @@ export const resourceNames = { }, }; +const createIndexAssetsOnce = once( + (logger: Logger, core: CoreSetup) => + pRetry(() => setupConversationAndKbIndexAssets({ logger, core })) +); + export class ObservabilityAIAssistantService { private readonly core: CoreSetup; private readonly logger: Logger; - private kbService?: KnowledgeBaseService; private config: ObservabilityAIAssistantConfig; - private readonly registrations: RegistrationCallback[] = []; constructor({ @@ -65,120 +65,8 @@ export class ObservabilityAIAssistantService { this.core = core; this.logger = logger; this.config = config; - - this.resetInit(); } - init = async () => {}; - - private resetInit = () => { - this.init = once(async () => { - return this.doInit().catch((error) => { - this.resetInit(); // reset the once flag if an error occurs - throw error; - }); - }); - }; - - private doInit = async () => { - try { - this.logger.debug('Setting up index assets'); - const [coreStart] = await this.core.getStartServices(); - - const { asInternalUser } = coreStart.elasticsearch.client; - - await asInternalUser.cluster.putComponentTemplate({ - create: false, - name: resourceNames.componentTemplate.conversations, - template: conversationComponentTemplate, - }); - - await asInternalUser.indices.putIndexTemplate({ - name: resourceNames.indexTemplate.conversations, - composed_of: [resourceNames.componentTemplate.conversations], - create: false, - index_patterns: [resourceNames.indexPatterns.conversations], - template: { - settings: { - number_of_shards: 1, - auto_expand_replicas: '0-1', - hidden: true, - }, - }, - }); - - const conversationAliasName = resourceNames.aliases.conversations; - - await createConcreteWriteIndex({ - esClient: asInternalUser, - logger: this.logger, - totalFieldsLimit: 10000, - indexPatterns: { - alias: conversationAliasName, - pattern: `${conversationAliasName}*`, - basePattern: `${conversationAliasName}*`, - name: `${conversationAliasName}-000001`, - template: resourceNames.indexTemplate.conversations, - }, - dataStreamAdapter: getDataStreamAdapter({ useDataStreamForAlerts: false }), - }); - - // Knowledge base: component template - await asInternalUser.cluster.putComponentTemplate({ - create: false, - name: resourceNames.componentTemplate.kb, - template: kbComponentTemplate, - }); - - // Knowledge base: index template - await asInternalUser.indices.putIndexTemplate({ - name: resourceNames.indexTemplate.kb, - composed_of: [resourceNames.componentTemplate.kb], - create: false, - index_patterns: [resourceNames.indexPatterns.kb], - template: { - settings: { - number_of_shards: 1, - auto_expand_replicas: '0-1', - hidden: true, - }, - }, - }); - - const kbAliasName = resourceNames.aliases.kb; - - // Knowledge base: write index - await createConcreteWriteIndex({ - esClient: asInternalUser, - logger: this.logger, - totalFieldsLimit: 10000, - indexPatterns: { - alias: kbAliasName, - pattern: `${kbAliasName}*`, - basePattern: `${kbAliasName}*`, - name: `${kbAliasName}-000001`, - template: resourceNames.indexTemplate.kb, - }, - dataStreamAdapter: getDataStreamAdapter({ useDataStreamForAlerts: false }), - }); - - this.kbService = new KnowledgeBaseService({ - core: this.core, - logger: this.logger.get('kb'), - config: this.config, - esClient: { - asInternalUser, - }, - }); - - this.logger.info('Successfully set up index assets'); - } catch (error) { - this.logger.error(`Failed setting up index assets: ${error.message}`); - this.logger.debug(error); - throw error; - } - }; - async getClient({ request, scopes, @@ -192,12 +80,11 @@ export class ObservabilityAIAssistantService { controller.abort(); }); - const [_, [coreStart, plugins]] = await Promise.all([ - this.init(), - this.core.getStartServices() as Promise< - [CoreStart, { security: SecurityPluginStart; actions: ActionsPluginStart }, unknown] - >, + const [[coreStart, plugins]] = await Promise.all([ + this.core.getStartServices(), + createIndexAssetsOnce(this.logger, this.core), ]); + // user will not be found when executed from system connector context const user = plugins.security.authc.getCurrentUser(request); @@ -207,12 +94,25 @@ export class ObservabilityAIAssistantService { const { spaceId } = getSpaceIdFromPath(basePath, coreStart.http.basePath.serverBasePath); + const { asInternalUser } = coreStart.elasticsearch.client; + + const kbService = new KnowledgeBaseService({ + core: this.core, + logger: this.logger.get('kb'), + config: this.config, + esClient: { + asInternalUser, + }, + }); + return new ObservabilityAIAssistantClient({ + core: this.core, + config: this.config, actionsClient: await plugins.actions.getActionsClientWithRequest(request), uiSettingsClient: coreStart.uiSettings.asScopedToClient(soClient), namespace: spaceId, esClient: { - asInternalUser: coreStart.elasticsearch.client.asInternalUser, + asInternalUser, asCurrentUser: coreStart.elasticsearch.client.asScoped(request).asCurrentUser, }, logger: this.logger, @@ -222,7 +122,7 @@ export class ObservabilityAIAssistantService { name: user.username, } : undefined, - knowledgeBaseService: this.kbService!, + knowledgeBaseService: kbService, scopes: scopes || ['all'], }); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/inference_endpoint.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/inference_endpoint.ts index e89028652d9ac..a2993f7353c61 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/inference_endpoint.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/inference_endpoint.ts @@ -9,6 +9,7 @@ import { errors } from '@elastic/elasticsearch'; import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; import moment from 'moment'; +import { ObservabilityAIAssistantConfig } from '../config'; export const AI_ASSISTANT_KB_INFERENCE_ID = 'obs_ai_assistant_kb_inference'; @@ -34,7 +35,7 @@ export async function createInferenceEndpoint({ service: 'elasticsearch', service_settings: { model_id: modelId, - adaptive_allocations: { enabled: true }, + adaptive_allocations: { enabled: true, min_number_of_allocations: 1 }, num_threads: 1, }, task_settings: {}, @@ -45,7 +46,7 @@ export async function createInferenceEndpoint({ } ); } catch (e) { - logger.error( + logger.debug( `Failed to create inference endpoint "${AI_ASSISTANT_KB_INFERENCE_ID}": ${e.message}` ); throw e; @@ -54,44 +55,30 @@ export async function createInferenceEndpoint({ export async function deleteInferenceEndpoint({ esClient, - logger, }: { esClient: { asCurrentUser: ElasticsearchClient; }; - logger: Logger; }) { - try { - const response = await esClient.asCurrentUser.inference.delete({ - inference_id: AI_ASSISTANT_KB_INFERENCE_ID, - force: true, - }); + const response = await esClient.asCurrentUser.inference.delete({ + inference_id: AI_ASSISTANT_KB_INFERENCE_ID, + force: true, + }); - return response; - } catch (e) { - logger.error(`Failed to delete inference endpoint: ${e.message}`); - throw e; - } + return response; } export async function getInferenceEndpoint({ esClient, - logger, }: { esClient: { asInternalUser: ElasticsearchClient }; - logger: Logger; }) { - try { - const response = await esClient.asInternalUser.inference.get({ - inference_id: AI_ASSISTANT_KB_INFERENCE_ID, - }); + const response = await esClient.asInternalUser.inference.get({ + inference_id: AI_ASSISTANT_KB_INFERENCE_ID, + }); - if (response.endpoints.length > 0) { - return response.endpoints[0]; - } - } catch (e) { - logger.error(`Failed to fetch inference endpoint: ${e.message}`); - throw e; + if (response.endpoints.length > 0) { + return response.endpoints[0]; } } @@ -102,3 +89,61 @@ export function isInferenceEndpointMissingOrUnavailable(error: Error) { error.body?.error?.type === 'status_exception') ); } + +export async function getElserModelStatus({ + esClient, + logger, + config, +}: { + esClient: { asInternalUser: ElasticsearchClient }; + logger: Logger; + config: ObservabilityAIAssistantConfig; +}) { + let errorMessage = ''; + const endpoint = await getInferenceEndpoint({ + esClient, + }).catch((error) => { + if (!isInferenceEndpointMissingOrUnavailable(error)) { + throw error; + } + errorMessage = error.message; + }); + + const enabled = config.enableKnowledgeBase; + if (!endpoint) { + return { ready: false, enabled, errorMessage }; + } + + const modelId = endpoint.service_settings?.model_id; + const modelStats = await esClient.asInternalUser.ml + .getTrainedModelsStats({ model_id: modelId }) + .catch((error) => { + logger.debug(`Failed to get model stats: ${error.message}`); + errorMessage = error.message; + }); + + if (!modelStats) { + return { ready: false, enabled, errorMessage }; + } + + const elserModelStats = modelStats.trained_model_stats.find( + (stats) => stats.deployment_stats?.deployment_id === AI_ASSISTANT_KB_INFERENCE_ID + ); + const deploymentState = elserModelStats?.deployment_stats?.state; + const allocationState = elserModelStats?.deployment_stats?.allocation_status.state; + const allocationCount = + elserModelStats?.deployment_stats?.allocation_status.allocation_count ?? 0; + const ready = + deploymentState === 'started' && allocationState === 'fully_allocated' && allocationCount > 0; + + return { + endpoint, + ready, + enabled, + model_stats: { + allocation_count: allocationCount, + deployment_state: deploymentState, + allocation_state: allocationState, + }, + }; +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts index 6cf89b0c9e22d..49e856db29d50 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts @@ -62,10 +62,6 @@ export const kbComponentTemplate: ClusterComponentTemplate['component_template'] semantic_text: { type: 'semantic_text', inference_id: AI_ASSISTANT_KB_INFERENCE_ID, - // @ts-expect-error: @elastic/elasticsearch does not have this type yet - model_settings: { - task_type: 'sparse_embedding', - }, }, 'ml.tokens': { type: 'rank_features', diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index fdde5ebb49970..1cf1cdc326fdf 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -20,10 +20,9 @@ import { import { getAccessQuery } from '../util/get_access_query'; import { getCategoryQuery } from '../util/get_category_query'; import { - AI_ASSISTANT_KB_INFERENCE_ID, createInferenceEndpoint, deleteInferenceEndpoint, - getInferenceEndpoint, + getElserModelStatus, isInferenceEndpointMissingOrUnavailable, } from '../inference_endpoint'; import { recallFromSearchConnectors } from './recall_from_search_connectors'; @@ -61,13 +60,13 @@ export class KnowledgeBaseService { }, modelId: string | undefined ) { - await deleteInferenceEndpoint({ esClient, logger: this.dependencies.logger }).catch((e) => {}); // ensure existing inference endpoint is deleted + await deleteInferenceEndpoint({ esClient }).catch((e) => {}); // ensure existing inference endpoint is deleted return createInferenceEndpoint({ esClient, logger: this.dependencies.logger, modelId }); } async reset(esClient: { asCurrentUser: ElasticsearchClient }) { try { - await deleteInferenceEndpoint({ esClient, logger: this.dependencies.logger }); + await deleteInferenceEndpoint({ esClient }); } catch (error) { if (isInferenceEndpointMissingOrUnavailable(error)) { return; @@ -437,58 +436,10 @@ export class KnowledgeBaseService { }; getStatus = async () => { - let errorMessage = ''; - const endpoint = await getInferenceEndpoint({ + return getElserModelStatus({ esClient: this.dependencies.esClient, logger: this.dependencies.logger, - }).catch((error) => { - if (!isInferenceEndpointMissingOrUnavailable(error)) { - throw error; - } - this.dependencies.logger.error(`Failed to get inference endpoint: ${error.message}`); - errorMessage = error.message; + config: this.dependencies.config, }); - - const enabled = this.dependencies.config.enableKnowledgeBase; - if (!endpoint) { - return { ready: false, enabled, errorMessage }; - } - - const modelId = endpoint.service_settings?.model_id; - const modelStats = await this.dependencies.esClient.asInternalUser.ml - .getTrainedModelsStats({ model_id: modelId }) - .catch((error) => { - this.dependencies.logger.error(`Failed to get model stats: ${error.message}`); - errorMessage = error.message; - }); - - if (!modelStats) { - return { ready: false, enabled, errorMessage }; - } - - const elserModelStats = modelStats.trained_model_stats.find( - (stats) => stats.deployment_stats?.deployment_id === AI_ASSISTANT_KB_INFERENCE_ID - ); - const deploymentState = elserModelStats?.deployment_stats?.state; - const allocationState = elserModelStats?.deployment_stats?.allocation_status.state; - const allocationCount = - elserModelStats?.deployment_stats?.allocation_status.allocation_count ?? 0; - const ready = - deploymentState === 'started' && allocationState === 'fully_allocated' && allocationCount > 0; - - this.dependencies.logger.debug( - `Model deployment state: ${deploymentState}, allocation state: ${allocationState}, ready: ${ready}` - ); - - return { - endpoint, - ready, - enabled, - model_stats: { - allocation_count: allocationCount, - deployment_state: deploymentState, - allocation_state: allocationState, - }, - }; }; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/setup_conversation_and_kb_index_assets.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/setup_conversation_and_kb_index_assets.ts new file mode 100644 index 0000000000000..30d55400bbbda --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/setup_conversation_and_kb_index_assets.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createConcreteWriteIndex, getDataStreamAdapter } from '@kbn/alerting-plugin/server'; +import type { CoreSetup, Logger } from '@kbn/core/server'; +import type { ObservabilityAIAssistantPluginStartDependencies } from '../types'; +import { conversationComponentTemplate } from './conversation_component_template'; +import { kbComponentTemplate } from './kb_component_template'; +import { resourceNames } from '.'; + +export async function setupConversationAndKbIndexAssets({ + logger, + core, +}: { + logger: Logger; + core: CoreSetup; +}) { + try { + logger.debug('Setting up index assets'); + const [coreStart] = await core.getStartServices(); + const { asInternalUser } = coreStart.elasticsearch.client; + + // Conversations: component template + await asInternalUser.cluster.putComponentTemplate({ + create: false, + name: resourceNames.componentTemplate.conversations, + template: conversationComponentTemplate, + }); + + // Conversations: index template + await asInternalUser.indices.putIndexTemplate({ + name: resourceNames.indexTemplate.conversations, + composed_of: [resourceNames.componentTemplate.conversations], + create: false, + index_patterns: [resourceNames.indexPatterns.conversations], + template: { + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + hidden: true, + }, + }, + }); + + // Conversations: write index + const conversationAliasName = resourceNames.aliases.conversations; + await createConcreteWriteIndex({ + esClient: asInternalUser, + logger, + totalFieldsLimit: 10000, + indexPatterns: { + alias: conversationAliasName, + pattern: `${conversationAliasName}*`, + basePattern: `${conversationAliasName}*`, + name: `${conversationAliasName}-000001`, + template: resourceNames.indexTemplate.conversations, + }, + dataStreamAdapter: getDataStreamAdapter({ useDataStreamForAlerts: false }), + }); + + // Knowledge base: component template + await asInternalUser.cluster.putComponentTemplate({ + create: false, + name: resourceNames.componentTemplate.kb, + template: kbComponentTemplate, + }); + + // Knowledge base: index template + await asInternalUser.indices.putIndexTemplate({ + name: resourceNames.indexTemplate.kb, + composed_of: [resourceNames.componentTemplate.kb], + create: false, + index_patterns: [resourceNames.indexPatterns.kb], + template: { + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + hidden: true, + }, + }, + }); + + // Knowledge base: write index + const kbAliasName = resourceNames.aliases.kb; + await createConcreteWriteIndex({ + esClient: asInternalUser, + logger, + totalFieldsLimit: 10000, + indexPatterns: { + alias: kbAliasName, + pattern: `${kbAliasName}*`, + basePattern: `${kbAliasName}*`, + name: `${kbAliasName}-000001`, + template: resourceNames.indexTemplate.kb, + }, + dataStreamAdapter: getDataStreamAdapter({ useDataStreamForAlerts: false }), + }); + + logger.info('Successfully set up index assets'); + } catch (error) { + logger.error(`Failed setting up index assets: ${error.message}`); + logger.debug(error); + } +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/task_manager_definitions/register_migrate_knowledge_base_entries_task.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/task_manager_definitions/register_migrate_knowledge_base_entries_task.ts index 3df125ab2ba2d..b75074dc7ea54 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/task_manager_definitions/register_migrate_knowledge_base_entries_task.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/task_manager_definitions/register_migrate_knowledge_base_entries_task.ts @@ -12,8 +12,10 @@ import type { CoreSetup, Logger } from '@kbn/core/server'; import pRetry from 'p-retry'; import { KnowledgeBaseEntry } from '../../../common'; import { resourceNames } from '..'; -import { getInferenceEndpoint } from '../inference_endpoint'; +import { getElserModelStatus } from '../inference_endpoint'; import { ObservabilityAIAssistantPluginStartDependencies } from '../../types'; +import { ObservabilityAIAssistantConfig } from '../../config'; +import { setupConversationAndKbIndexAssets } from '../setup_conversation_and_kb_index_assets'; const TASK_ID = 'obs-ai-assistant:knowledge-base-migration-task-id'; const TASK_TYPE = 'obs-ai-assistant:knowledge-base-migration'; @@ -25,36 +27,66 @@ export async function registerMigrateKnowledgeBaseEntriesTask({ taskManager, logger, core, + config, }: { taskManager: TaskManagerSetupContract; logger: Logger; core: CoreSetup; + config: ObservabilityAIAssistantConfig; }) { - logger.debug(`Register task "${TASK_TYPE}"`); - const [coreStart, pluginsStart] = await core.getStartServices(); - taskManager.registerTaskDefinitions({ - [TASK_TYPE]: { - title: 'Migrate AI Assistant Knowledge Base', - description: `Migrates AI Assistant knowledge base entries`, - timeout: '1h', - maxAttempts: 5, - createTaskRunner() { - return { - async run() { - logger.debug(`Run task: "${TASK_TYPE}"`); - - const esClient = { asInternalUser: coreStart.elasticsearch.client.asInternalUser }; - await runSemanticTextKnowledgeBaseMigration({ esClient, logger }); - }, - }; + try { + logger.debug(`Register task "${TASK_TYPE}"`); + taskManager.registerTaskDefinitions({ + [TASK_TYPE]: { + title: 'Migrate AI Assistant Knowledge Base', + description: `Migrates AI Assistant knowledge base entries`, + timeout: '1h', + maxAttempts: 5, + createTaskRunner() { + return { + async run() { + logger.debug(`Run task: "${TASK_TYPE}"`); + const esClient = coreStart.elasticsearch.client; + + const hasKbIndex = await esClient.asInternalUser.indices.exists({ + index: resourceNames.aliases.kb, + }); + + if (!hasKbIndex) { + logger.debug( + 'Knowledge base index does not exist. Skipping semantic text migration.' + ); + return; + } + + // update fields and mappings + await setupConversationAndKbIndexAssets({ logger, core }); + + // run migration + await runSemanticTextKnowledgeBaseMigration({ esClient, logger, config }); + }, + }; + }, }, - }, - }); + }); + } catch (error) { + logger.error(`Failed to register task "${TASK_TYPE}". Error: ${error}`); + } + + try { + logger.debug(`Scheduled task: "${TASK_TYPE}"`); + await scheduleSemanticTextMigration(pluginsStart); + } catch (error) { + logger.error(`Failed to schedule task "${TASK_TYPE}". Error: ${error}`); + } +} - logger.debug(`Scheduled task: "${TASK_TYPE}"`); - await pluginsStart.taskManager.ensureScheduled({ +export function scheduleSemanticTextMigration( + pluginsStart: ObservabilityAIAssistantPluginStartDependencies +) { + return pluginsStart.taskManager.ensureScheduled({ id: TASK_ID, taskType: TASK_TYPE, scope: ['aiAssistant'], @@ -66,9 +98,11 @@ export async function registerMigrateKnowledgeBaseEntriesTask({ export async function runSemanticTextKnowledgeBaseMigration({ esClient, logger, + config, }: { esClient: { asInternalUser: ElasticsearchClient }; logger: Logger; + config: ObservabilityAIAssistantConfig; }) { logger.debug('Knowledge base migration: Running migration'); @@ -98,7 +132,7 @@ export async function runSemanticTextKnowledgeBaseMigration({ logger.debug(`Knowledge base migration: Found ${response.hits.hits.length} entries to migrate`); - await waitForInferenceEndpoint({ esClient, logger }); + await waitForModel({ esClient, logger, config }); // Limit the number of concurrent requests to avoid overloading the cluster const limiter = pLimit(10); @@ -109,6 +143,7 @@ export async function runSemanticTextKnowledgeBaseMigration({ } return esClient.asInternalUser.update({ + refresh: 'wait_for', index: resourceNames.aliases.kb, id: hit._id, body: { @@ -123,27 +158,29 @@ export async function runSemanticTextKnowledgeBaseMigration({ await Promise.all(promises); logger.debug(`Knowledge base migration: Migrated ${promises.length} entries`); - await runSemanticTextKnowledgeBaseMigration({ esClient, logger }); + await runSemanticTextKnowledgeBaseMigration({ esClient, logger, config }); } catch (e) { - logger.error('Knowledge base migration: Failed to migrate entries'); - logger.error(e); + logger.error(`Knowledge base migration failed: ${e.message}`); } } -async function waitForInferenceEndpoint({ +async function waitForModel({ esClient, logger, + config, }: { esClient: { asInternalUser: ElasticsearchClient }; logger: Logger; + config: ObservabilityAIAssistantConfig; }) { return pRetry( async () => { - const endpoint = await getInferenceEndpoint({ esClient, logger }); - if (!endpoint) { - throw new Error('Inference endpoint not yet ready'); + const { ready } = await getElserModelStatus({ esClient, logger, config }); + if (!ready) { + logger.debug('Elser model is not yet ready. Retrying...'); + throw new Error('Elser model is not yet ready'); } }, - { retries: 20, factor: 2 } + { retries: 30, factor: 2, maxTimeout: 30_000 } ); } From 1edcb62a30597104bfead7864b5c52dbb1086511 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:51:11 +0100 Subject: [PATCH 07/51] [Fleet] Rollover datastreams on subobjects mapper exception (#202689) ## Summary Closes https://github.com/elastic/kibana/issues/193044 The previously added `subobjectsFieldChanged` check was not picking up the subobjects change in this case (okta integration), maybe because it is in dynamic template mappings. Added an additional condition to pick up on the `mapper_exception` with subobjects mentioned in the reason. To verify: - install okta-2.11.0 integration - upgrade to okta-2.12.0 - expect upgrade successful, the data stream should be rolled over ``` POST kbn:/api/fleet/epm/packages/okta/2.11.0 { "force": true } POST logs-okta.system-default/_doc { "message": "abc", "@timestamp": "2024-05-30T07:50:00.000Z" } POST kbn:/api/fleet/epm/packages/okta/2.12.0 { "force": true } GET logs-okta.system-default/_mapping ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../elasticsearch/template/template.test.ts | 51 +++++++++++++++++++ .../epm/elasticsearch/template/template.ts | 4 ++ 2 files changed, 55 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index dbc93e35c8218..9fc5383902ff3 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -2105,6 +2105,57 @@ describe('EPM template', () => { }) ); }); + it('should rollover on mapper exception with subobjects in reason', async () => { + const esClient = elasticsearchServiceMock.createElasticsearchClient(); + esClient.indices.getDataStream.mockResponse({ + data_streams: [{ name: 'test.prefix1-default' }], + } as any); + esClient.indices.get.mockResponse({ + 'test.prefix1-default': { + mappings: {}, + }, + } as any); + esClient.indices.simulateTemplate.mockResponse({ + template: { + settings: { index: {} }, + mappings: {}, + }, + } as any); + esClient.indices.putMapping.mockImplementation(() => { + throw new errors.ResponseError({ + body: { + error: { + type: 'mapper_exception', + reason: + "the [subobjects] parameter can't be updated for the object mapping [okta.debug_context.debug_data]", + }, + }, + } as any); + }); + + const logger = loggerMock.create(); + await updateCurrentWriteIndices(esClient, logger, [ + { + templateName: 'test', + indexTemplate: { + index_patterns: ['test.*-*'], + template: { + settings: { index: {} }, + mappings: {}, + }, + } as any, + }, + ]); + + expect(esClient.transport.request).toHaveBeenCalledWith( + expect.objectContaining({ + path: '/test.prefix1-default/_rollover', + querystring: { + lazy: true, + }, + }) + ); + }); it('should skip rollover on expected error when flag is on', async () => { const esClient = elasticsearchServiceMock.createElasticsearchClient(); esClient.indices.getDataStream.mockResponse({ diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index b6d357193b14d..57ce30550af68 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -1084,6 +1084,10 @@ const updateExistingDataStream = async ({ // if update fails, rollover data stream and bail out } catch (err) { + subobjectsFieldChanged = + subobjectsFieldChanged || + (err.body?.error?.type === 'mapper_exception' && + err.body?.error?.reason?.includes('subobjects')); if ( (isResponseError(err) && err.statusCode === 400 && From ded18eeaac82787ea57d4991bad28b983d43ad0c Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Tue, 3 Dec 2024 15:29:28 +0100 Subject: [PATCH 08/51] [ML] Trained Models: Fixes spaces sync to retrieve 10000 models (#202712) ## Summary The default page size for the /trained_models API is 100. As a result, the spaces sync task only fetched the first 100 models, leaving the rest unassigned to spaces and therefore invisible in the ML UI. This PR increases the page size to 10,000 to ensure all models are properly assigned to Kibana spaces. ### Checklist - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- x-pack/plugins/ml/common/constants/trained_models.ts | 12 ++++++++++++ .../models/data_frame_analytics/analytics_manager.ts | 2 +- x-pack/plugins/ml/server/routes/trained_models.ts | 3 +-- x-pack/plugins/ml/server/saved_objects/util.ts | 5 ++++- 4 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/ml/common/constants/trained_models.ts diff --git a/x-pack/plugins/ml/common/constants/trained_models.ts b/x-pack/plugins/ml/common/constants/trained_models.ts new file mode 100644 index 0000000000000..c6092037cbfb4 --- /dev/null +++ b/x-pack/plugins/ml/common/constants/trained_models.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Default page for the trained_models endpoint is 100, + * which is too small for the most cases, so we set it to 10000. + */ +export const DEFAULT_TRAINED_MODELS_PAGE_SIZE = 10000; diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts index 4e112a2ff313b..e720f12fa4dd5 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts @@ -20,6 +20,7 @@ import { } from '@kbn/ml-data-frame-analytics-utils'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import type { CloudSetup } from '@kbn/cloud-plugin/server'; +import { DEFAULT_TRAINED_MODELS_PAGE_SIZE } from '../../../common/constants/trained_models'; import type { MlFeatures } from '../../../common/constants/app'; import type { ModelService } from '../model_management/models_provider'; import { modelsProvider } from '../model_management'; @@ -38,7 +39,6 @@ import { isTransformLinkReturnType, } from './types'; import type { MlClient } from '../../lib/ml_client'; -import { DEFAULT_TRAINED_MODELS_PAGE_SIZE } from '../../routes/trained_models'; export class AnalyticsManager { private _trainedModels: estypes.MlTrainedModelConfig[] = []; diff --git a/x-pack/plugins/ml/server/routes/trained_models.ts b/x-pack/plugins/ml/server/routes/trained_models.ts index c0010777ecf18..2782c4be18207 100644 --- a/x-pack/plugins/ml/server/routes/trained_models.ts +++ b/x-pack/plugins/ml/server/routes/trained_models.ts @@ -17,6 +17,7 @@ import type { } from '@kbn/ml-trained-models-utils'; import { isDefined } from '@kbn/ml-is-defined'; import type { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { DEFAULT_TRAINED_MODELS_PAGE_SIZE } from '../../common/constants/trained_models'; import { type MlFeatures, ML_INTERNAL_BASE_PATH } from '../../common/constants/app'; import type { RouteInitialization } from '../types'; import { wrapError } from '../client/error_wrapper'; @@ -44,8 +45,6 @@ import { mlLog } from '../lib/log'; import { forceQuerySchema } from './schemas/anomaly_detectors_schema'; import { modelsProvider } from '../models/model_management'; -export const DEFAULT_TRAINED_MODELS_PAGE_SIZE = 10000; - export function filterForEnabledFeatureModels< T extends TrainedModelConfigResponse | estypes.MlTrainedModelConfig >(models: T[], enabledFeatures: MlFeatures) { diff --git a/x-pack/plugins/ml/server/saved_objects/util.ts b/x-pack/plugins/ml/server/saved_objects/util.ts index 27562e919b79f..7f3f087ea61be 100644 --- a/x-pack/plugins/ml/server/saved_objects/util.ts +++ b/x-pack/plugins/ml/server/saved_objects/util.ts @@ -12,6 +12,7 @@ import { type IScopedClusterClient, SavedObjectsClient, } from '@kbn/core/server'; +import { DEFAULT_TRAINED_MODELS_PAGE_SIZE } from '../../common/constants/trained_models'; import type { TrainedModelJob, MLSavedObjectService } from './service'; import { ML_JOB_SAVED_OBJECT_TYPE } from '../../common/types/saved_objects'; @@ -86,7 +87,9 @@ export function mlFunctionsFactory(client: IScopedClusterClient) { }, async getTrainedModels() { try { - return await client.asInternalUser.ml.getTrainedModels(); + return await client.asInternalUser.ml.getTrainedModels({ + size: DEFAULT_TRAINED_MODELS_PAGE_SIZE, + }); } catch (error) { return null; } From 27f650bf990c84a1de0c46d4867923773a8d4166 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 3 Dec 2024 15:47:30 +0100 Subject: [PATCH 09/51] [chore] update playwright version + ownership (#202535) ## Summary Since the new Kibana test framework has a strong dependency on `@playwright/test`, moving it under appex-qa ownership I had to update and explicitly specify types when extending the pre-existing fixtures as there was a bug fix enforcing it https://github.com/microsoft/playwright/pull/32066 --- package.json | 6 +-- .../src/playwright/fixtures/test/page.ts | 23 +++++----- renovate.json | 6 +-- .../ui_tests/fixtures/index.ts | 11 ++++- yarn.lock | 42 +++++++++---------- 5 files changed, 50 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index f6c3330adad25..eb82728877813 100644 --- a/package.json +++ b/package.json @@ -1515,7 +1515,7 @@ "@mswjs/http-middleware": "^0.10.1", "@octokit/rest": "^17.11.2", "@parcel/watcher": "^2.1.0", - "@playwright/test": "=1.46.0", + "@playwright/test": "1.49.0", "@redocly/cli": "^1.25.14", "@statoscope/webpack-plugin": "^5.28.2", "@storybook/addon-a11y": "^6.5.16", @@ -1803,8 +1803,8 @@ "pirates": "^4.0.1", "piscina": "^3.2.0", "pixelmatch": "^5.3.0", - "playwright": "=1.46.0", - "playwright-chromium": "=1.46.0", + "playwright": "1.49.0", + "playwright-chromium": "1.49.0", "pngjs": "^7.0.0", "postcss": "^8.4.31", "postcss-loader": "^4.2.0", diff --git a/packages/kbn-scout/src/playwright/fixtures/test/page.ts b/packages/kbn-scout/src/playwright/fixtures/test/page.ts index b41b8a92f2701..4cab61e4ffec5 100644 --- a/packages/kbn-scout/src/playwright/fixtures/test/page.ts +++ b/packages/kbn-scout/src/playwright/fixtures/test/page.ts @@ -9,7 +9,7 @@ import { Page, test as base } from '@playwright/test'; import { subj } from '@kbn/test-subj-selector'; -import { ScoutPage, KibanaUrl } from '../types'; +import { ScoutPage, KibanaUrl, ScoutTestFixtures, ScoutWorkerFixtures } from '../types'; /** * Instead of defining each method individually, we use a list of method names and loop through them, creating methods dynamically. @@ -95,17 +95,20 @@ function extendPageWithTestSubject(page: Page): ScoutPage['testSubj'] { * await page.gotoApp('discover); * ``` */ -export const scoutPageFixture = base.extend<{ page: ScoutPage; kbnUrl: KibanaUrl }>({ - page: async ({ page, kbnUrl }, use) => { +export const scoutPageFixture = base.extend({ + page: async ( + { page, kbnUrl }: { page: Page; kbnUrl: KibanaUrl }, + use: (extendedPage: ScoutPage) => Promise + ) => { + const extendedPage = page as ScoutPage; // Extend page with '@kbn/test-subj-selector' support - page.testSubj = extendPageWithTestSubject(page); - + extendedPage.testSubj = extendPageWithTestSubject(page); // Method to navigate to specific Kibana apps - page.gotoApp = (appName: string) => page.goto(kbnUrl.app(appName)); - - page.waitForLoadingIndicatorHidden = () => - page.testSubj.waitForSelector('globalLoadingIndicator-hidden', { state: 'attached' }); + extendedPage.gotoApp = (appName: string) => page.goto(kbnUrl.app(appName)); + // Method to wait for global loading indicator to be hidden + extendedPage.waitForLoadingIndicatorHidden = () => + extendedPage.testSubj.waitForSelector('globalLoadingIndicator-hidden', { state: 'attached' }); - await use(page); + await use(extendedPage); }, }); diff --git a/renovate.json b/renovate.json index c9ccd3969e3d6..b010d5465dc33 100644 --- a/renovate.json +++ b/renovate.json @@ -144,7 +144,9 @@ "nyc", "oboe", "pixelmatch", + "@playwright/test", "playwright", + "playwright-chromium", "pngjs", "sharp", "superagent", @@ -1327,9 +1329,7 @@ { "groupName": "Security Engineering Productivity", "matchDepNames": [ - "dotenv", - "playwright-chromium", - "@playwright/test" + "dotenv" ], "reviewers": [ "team:security-engineering-productivity" diff --git a/x-pack/plugins/discover_enhanced/ui_tests/fixtures/index.ts b/x-pack/plugins/discover_enhanced/ui_tests/fixtures/index.ts index 4cff802b3b89f..cf12a98368b90 100644 --- a/x-pack/plugins/discover_enhanced/ui_tests/fixtures/index.ts +++ b/x-pack/plugins/discover_enhanced/ui_tests/fixtures/index.ts @@ -21,7 +21,16 @@ export interface ExtendedScoutTestFixtures extends ScoutTestFixtures { } export const test = base.extend({ - pageObjects: async ({ pageObjects, page }, use) => { + pageObjects: async ( + { + pageObjects, + page, + }: { + pageObjects: ExtendedScoutTestFixtures['pageObjects']; + page: ExtendedScoutTestFixtures['page']; + }, + use: (pageObjects: ExtendedScoutTestFixtures['pageObjects']) => Promise + ) => { const extendedPageObjects = { ...pageObjects, demo: createLazyPageObject(DemoPage, page), diff --git a/yarn.lock b/yarn.lock index 90429bc21f489..70875c1d31374 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8986,12 +8986,12 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== -"@playwright/test@=1.46.0": - version "1.46.0" - resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.46.0.tgz#ccea6d22c40ee7fa567e4192fafbdf2a907e2714" - integrity sha512-/QYft5VArOrGRP5pgkrfKksqsKA6CEFyGQ/gjNe6q0y4tZ1aaPfq4gIjudr1s3D+pXyrPRdsy4opKDrjBabE5w== +"@playwright/test@1.49.0": + version "1.49.0" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.49.0.tgz#74227385b58317ee076b86b56d0e1e1b25cff01e" + integrity sha512-DMulbwQURa8rNIQrf94+jPJQ4FmOVdpE5ZppRNvWVjvhC+6sOeo28r8MgIpQRYouXRtt/FCCXU7zn20jnHR4Qw== dependencies: - playwright "1.46.0" + playwright "1.49.0" "@pmmmwh/react-refresh-webpack-plugin@^0.5.3": version "0.5.7" @@ -26212,6 +26212,13 @@ platform@^1.3.0: resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444" integrity sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q== +playwright-chromium@1.49.0: + version "1.49.0" + resolved "https://registry.yarnpkg.com/playwright-chromium/-/playwright-chromium-1.49.0.tgz#0661428204396dbf5445eb04536d43a5f91bca59" + integrity sha512-xU+nOHawNFKfJsHTTGyWqSJ5nRGGHQq1wTsc49H9rM+hDNnoKZi+3m12mGoLpqvJP7vRjZQ3uvU9/UJZbrJ1AA== + dependencies: + playwright-core "1.49.0" + playwright-chromium@=1.45.1: version "1.45.1" resolved "https://registry.yarnpkg.com/playwright-chromium/-/playwright-chromium-1.45.1.tgz#a20b513edbc0435b2e06a303aac61001f44bf094" @@ -26219,29 +26226,22 @@ playwright-chromium@=1.45.1: dependencies: playwright-core "1.45.1" -playwright-chromium@=1.46.0: - version "1.46.0" - resolved "https://registry.yarnpkg.com/playwright-chromium/-/playwright-chromium-1.46.0.tgz#f24228fec92b380ccc8f5f365b897e9d88b612f6" - integrity sha512-UTHYZsr49XFYRQkpCfaHxL63vfu6uThxR1DrNwnU6qik/OworFcugTOJMWFMoop3QP+ThU8laAMumauLdLZXCQ== - dependencies: - playwright-core "1.46.0" - playwright-core@1.45.1, playwright-core@=1.45.1: version "1.45.1" resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.45.1.tgz#549a2701556b58245cc75263f9fc2795c1158dc1" integrity sha512-LF4CUUtrUu2TCpDw4mcrAIuYrEjVDfT1cHbJMfwnE2+1b8PZcFzPNgvZCvq2JfQ4aTjRCCHw5EJ2tmr2NSzdPg== -playwright-core@1.46.0: - version "1.46.0" - resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.46.0.tgz#2336ac453a943abf0dc95a76c117f9d3ebd390eb" - integrity sha512-9Y/d5UIwuJk8t3+lhmMSAJyNP1BUC/DqP3cQJDQQL/oWqAiuPTLgy7Q5dzglmTLwcBRdetzgNM/gni7ckfTr6A== +playwright-core@1.49.0: + version "1.49.0" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.49.0.tgz#8e69ffed3f41855b854982f3632f2922c890afcb" + integrity sha512-R+3KKTQF3npy5GTiKH/T+kdhoJfJojjHESR1YEWhYuEKRVfVaxH3+4+GvXE5xyCngCxhxnykk0Vlah9v8fs3jA== -playwright@1.46.0, playwright@=1.46.0: - version "1.46.0" - resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.46.0.tgz#c7ff490deae41fc1e814bf2cb62109dd9351164d" - integrity sha512-XYJ5WvfefWONh1uPAUAi0H2xXV5S3vrtcnXe6uAOgdGi3aSpqOSXX08IAjXW34xitfuOJsvXU5anXZxPSEQiJw== +playwright@1.49.0: + version "1.49.0" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.49.0.tgz#df6b9e05423377a99658202844a294a8afb95d0a" + integrity sha512-eKpmys0UFDnfNb3vfsf8Vx2LEOtflgRebl0Im2eQQnYMA4Aqd+Zw8bEOB+7ZKvN76901mRnqdsiOGKxzVTbi7A== dependencies: - playwright-core "1.46.0" + playwright-core "1.49.0" optionalDependencies: fsevents "2.3.2" From 6ef0284bce0e2fb86d2de6377916dce647aace26 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula <123897612+bhapas@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:56:10 +0100 Subject: [PATCH 10/51] [Automatic Import] Add base for ftr api tests (#200169) ## Summary This PR adds a baseline for FTR API tests for Automatic Import. - Relates https://github.com/elastic/kibana/issues/196063 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/ftr_security_stateful_configs.yml | 1 + .github/CODEOWNERS | 3 + .../common/config.ts | 89 +++++++++++++++++++ .../common/ftr_provider_context.d.ts | 12 +++ .../common/lib/api/analyze_logs.ts | 36 ++++++++ .../common/lib/api/categorization.ts | 36 ++++++++ .../common/lib/api/ecs.ts | 36 ++++++++ .../common/lib/api/index.ts | 7 ++ .../common/lib/api/related.ts | 36 ++++++++ .../common/lib/api/user_profiles.ts | 38 ++++++++ .../common/lib/authentication/index.ts | 85 ++++++++++++++++++ .../common/lib/authentication/roles.ts | 56 ++++++++++++ .../common/lib/authentication/types.ts | 54 +++++++++++ .../common/lib/authentication/users.ts | 29 ++++++ .../common/services.ts | 8 ++ .../security/config_basic.ts | 16 ++++ .../tests/basic/graphs/analyze_logs.ts | 34 +++++++ .../tests/basic/graphs/categorization.ts | 38 ++++++++ .../security/tests/basic/graphs/ecs.ts | 35 ++++++++ .../security/tests/basic/graphs/related.ts | 38 ++++++++ .../security/tests/basic/index.ts | 32 +++++++ x-pack/test/tsconfig.json | 1 + 22 files changed, 720 insertions(+) create mode 100644 x-pack/test/automatic_import_api_integration/common/config.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/ftr_provider_context.d.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/api/analyze_logs.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/api/categorization.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/api/ecs.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/api/index.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/api/related.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/api/user_profiles.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/authentication/index.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/authentication/roles.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/authentication/types.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/lib/authentication/users.ts create mode 100644 x-pack/test/automatic_import_api_integration/common/services.ts create mode 100644 x-pack/test/automatic_import_api_integration/security/config_basic.ts create mode 100644 x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/analyze_logs.ts create mode 100644 x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/categorization.ts create mode 100644 x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/ecs.ts create mode 100644 x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/related.ts create mode 100644 x-pack/test/automatic_import_api_integration/security/tests/basic/index.ts diff --git a/.buildkite/ftr_security_stateful_configs.yml b/.buildkite/ftr_security_stateful_configs.yml index 46c6f356b3ae4..bd8bc9e922f0d 100644 --- a/.buildkite/ftr_security_stateful_configs.yml +++ b/.buildkite/ftr_security_stateful_configs.yml @@ -101,3 +101,4 @@ enabled: - x-pack/test/cloud_security_posture_functional/config.ts - x-pack/test/cloud_security_posture_functional/config.agentless.ts - x-pack/test/cloud_security_posture_functional/data_views/config.ts + - x-pack/test/automatic_import_api_integration/security/config_basic.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f5c2ed37db761..51ab02111debd 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2459,6 +2459,9 @@ x-pack/plugins/security_solution/common/api/entity_analytics @elastic/security-e ## Security Solution sub teams - GenAI x-pack/test/security_solution_api_integration/test_suites/genai @elastic/security-generative-ai +## Security Solution sub teams - Automatic Import +x-pack/test/automatic_import_api_integration @elastic/security-scalability + # Security Defend Workflows - OSQuery Ownership /x-pack/test/osquery_cypress @elastic/security-defend-workflows /x-pack/plugins/osquery @elastic/security-defend-workflows diff --git a/x-pack/test/automatic_import_api_integration/common/config.ts b/x-pack/test/automatic_import_api_integration/common/config.ts new file mode 100644 index 0000000000000..aa83401a5e8b6 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/config.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CA_CERT_PATH } from '@kbn/dev-utils'; +import { FtrConfigProviderContext } from '@kbn/test'; +import { services } from './services'; + +interface CreateTestConfigOptions { + license: string; + disabledPlugins?: string[]; + ssl?: boolean; + testFiles?: string[]; + publicBaseUrl?: boolean; +} + +const enabledActionTypes = ['.bedrock', '.gemini', '.gen-ai']; + +export function createTestConfig(name: string, options: CreateTestConfigOptions) { + const { license = 'trial', disabledPlugins = [], ssl = false, testFiles = [] } = options; + + return async ({ readConfigFile }: FtrConfigProviderContext) => { + const xPackApiIntegrationTestsConfig = await readConfigFile( + require.resolve('../../api_integration/config.ts') + ); + + const servers = { + ...xPackApiIntegrationTestsConfig.get('servers'), + elasticsearch: { + ...xPackApiIntegrationTestsConfig.get('servers.elasticsearch'), + protocol: ssl ? 'https' : 'http', + }, + }; + + return { + testFiles, + servers, + services, + junit: { + reportName: 'X-Pack Automatic Import API Integration Tests', + }, + esTestCluster: { + ...xPackApiIntegrationTestsConfig.get('esTestCluster'), + license, + ssl, + serverArgs: [ + `xpack.license.self_generated.type=${license}`, + `xpack.security.enabled=${ + !disabledPlugins.includes('security') && ['trial', 'basic'].includes(license) + }`, + ], + }, + kbnTestServer: { + ...xPackApiIntegrationTestsConfig.get('kbnTestServer'), + serverArgs: [ + ...xPackApiIntegrationTestsConfig.get('kbnTestServer.serverArgs'), + ...(options.publicBaseUrl ? ['--server.publicBaseUrl=https://localhost:5601'] : []), + `--xpack.actions.allowedHosts=${JSON.stringify(['localhost', 'some.non.existent.com'])}`, + `--xpack.actions.enabledActionTypes=${JSON.stringify(enabledActionTypes)}`, + '--xpack.eventLog.logEntries=true', + // Configure a bedrock connector as a default + `--xpack.actions.preconfigured=${JSON.stringify({ + 'preconfigured-bedrock': { + name: 'preconfigured-bedrock', + actionTypeId: '.bedrock', + config: { + apiUrl: 'https://example.com', + }, + secrets: { + username: 'elastic', + password: 'elastic', + }, + }, + })}`, + ...(ssl + ? [ + `--elasticsearch.hosts=${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`, + `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, + ] + : []), + '--xpack.integration_assistant.enabled=true', + ], + }, + }; + }; +} diff --git a/x-pack/test/automatic_import_api_integration/common/ftr_provider_context.d.ts b/x-pack/test/automatic_import_api_integration/common/ftr_provider_context.d.ts new file mode 100644 index 0000000000000..aa56557c09df8 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/ftr_provider_context.d.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { GenericFtrProviderContext } from '@kbn/test'; + +import { services } from './services'; + +export type FtrProviderContext = GenericFtrProviderContext; diff --git a/x-pack/test/automatic_import_api_integration/common/lib/api/analyze_logs.ts b/x-pack/test/automatic_import_api_integration/common/lib/api/analyze_logs.ts new file mode 100644 index 0000000000000..4ad3034a0f55b --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/api/analyze_logs.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type SuperTest from 'supertest'; +import { + AnalyzeLogsRequestBody, + ANALYZE_LOGS_PATH, + AnalyzeLogsResponse, +} from '@kbn/integration-assistant-plugin/common'; +import { superUser } from '../authentication/users'; +import { User } from '../authentication/types'; + +export const postAnalyzeLogs = async ({ + supertest, + req, + expectedHttpCode = 404, + auth = { user: superUser }, +}: { + supertest: SuperTest.Agent; + req: AnalyzeLogsRequestBody; + expectedHttpCode?: number; + auth: { user: User }; +}): Promise => { + const { body: response } = await supertest + .post(`${ANALYZE_LOGS_PATH}`) + .send(req) + .set('kbn-xsrf', 'abc') + .set('elastic-api-version', '1') + .auth(auth.user.username, auth.user.password) + .expect(expectedHttpCode); + + return response; +}; diff --git a/x-pack/test/automatic_import_api_integration/common/lib/api/categorization.ts b/x-pack/test/automatic_import_api_integration/common/lib/api/categorization.ts new file mode 100644 index 0000000000000..f14efc63ee8a0 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/api/categorization.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type SuperTest from 'supertest'; +import { + CategorizationRequestBody, + CATEGORIZATION_GRAPH_PATH, + CategorizationResponse, +} from '@kbn/integration-assistant-plugin/common'; +import { superUser } from '../authentication/users'; +import { User } from '../authentication/types'; + +export const postCategorization = async ({ + supertest, + req, + expectedHttpCode = 404, + auth = { user: superUser }, +}: { + supertest: SuperTest.Agent; + req: CategorizationRequestBody; + expectedHttpCode?: number; + auth: { user: User }; +}): Promise => { + const { body: response } = await supertest + .post(`${CATEGORIZATION_GRAPH_PATH}`) + .send(req) + .set('kbn-xsrf', 'abc') + .set('elastic-api-version', '1') + .auth(auth.user.username, auth.user.password) + .expect(expectedHttpCode); + + return response; +}; diff --git a/x-pack/test/automatic_import_api_integration/common/lib/api/ecs.ts b/x-pack/test/automatic_import_api_integration/common/lib/api/ecs.ts new file mode 100644 index 0000000000000..3169659d7e2f3 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/api/ecs.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type SuperTest from 'supertest'; +import { + EcsMappingRequestBody, + ECS_GRAPH_PATH, + EcsMappingResponse, +} from '@kbn/integration-assistant-plugin/common'; +import { superUser } from '../authentication/users'; +import { User } from '../authentication/types'; + +export const postEcsMapping = async ({ + supertest, + req, + expectedHttpCode = 404, + auth = { user: superUser }, +}: { + supertest: SuperTest.Agent; + req: EcsMappingRequestBody; + expectedHttpCode?: number; + auth: { user: User }; +}): Promise => { + const { body: response } = await supertest + .post(`${ECS_GRAPH_PATH}`) + .send(req) + .set('kbn-xsrf', 'abc') + .set('elastic-api-version', '1') + .auth(auth.user.username, auth.user.password) + .expect(expectedHttpCode); + + return response; +}; diff --git a/x-pack/test/automatic_import_api_integration/common/lib/api/index.ts b/x-pack/test/automatic_import_api_integration/common/lib/api/index.ts new file mode 100644 index 0000000000000..d07ba14d384fd --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/api/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './user_profiles'; diff --git a/x-pack/test/automatic_import_api_integration/common/lib/api/related.ts b/x-pack/test/automatic_import_api_integration/common/lib/api/related.ts new file mode 100644 index 0000000000000..7ebe3d3bf89c2 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/api/related.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type SuperTest from 'supertest'; +import { + RelatedRequestBody, + RELATED_GRAPH_PATH, + RelatedResponse, +} from '@kbn/integration-assistant-plugin/common'; +import { superUser } from '../authentication/users'; +import { User } from '../authentication/types'; + +export const postRelated = async ({ + supertest, + req, + expectedHttpCode = 404, + auth = { user: superUser }, +}: { + supertest: SuperTest.Agent; + req: RelatedRequestBody; + expectedHttpCode?: number; + auth: { user: User }; +}): Promise => { + const { body: response } = await supertest + .post(`${RELATED_GRAPH_PATH}`) + .send(req) + .set('kbn-xsrf', 'abc') + .set('elastic-api-version', '1') + .auth(auth.user.username, auth.user.password) + .expect(expectedHttpCode); + + return response; +}; diff --git a/x-pack/test/automatic_import_api_integration/common/lib/api/user_profiles.ts b/x-pack/test/automatic_import_api_integration/common/lib/api/user_profiles.ts new file mode 100644 index 0000000000000..4a5c083e00fd6 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/api/user_profiles.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; +import { parse as parseCookie, Cookie } from 'tough-cookie'; +import { superUser } from '../authentication/users'; +import { User } from '../authentication/types'; + +export const loginUsers = async ({ + supertest, + users = [superUser], +}: { + supertest: SuperTest.Agent; + users?: User[]; +}) => { + const cookies: Cookie[] = []; + + for (const user of users) { + const response = await supertest + .post('/internal/security/login') + .set('kbn-xsrf', 'xxx') + .send({ + providerType: 'basic', + providerName: 'basic', + currentURL: '/', + params: { username: user.username, password: user.password }, + }) + .expect(200); + + cookies.push(parseCookie(response.header['set-cookie'][0])!); + } + + return cookies; +}; diff --git a/x-pack/test/automatic_import_api_integration/common/lib/authentication/index.ts b/x-pack/test/automatic_import_api_integration/common/lib/authentication/index.ts new file mode 100644 index 0000000000000..a886cf111d3f3 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/authentication/index.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext as CommonFtrProviderContext } from '../../ftr_provider_context'; +import { Role, User, UserInfo } from './types'; +import { noIntegrationsUser, users } from './users'; +import { roles } from './roles'; +import { loginUsers } from '../api'; + +export const getUserInfo = (user: User): UserInfo => ({ + username: user.username, + full_name: user.username.replace('_', ' '), + email: `${user.username}@elastic.co`, +}); + +/** + * Creates the users and roles for use in the tests. Defaults to specific users and roles used by the security + * scenarios but can be passed specific ones as well. + */ +export const createUsersAndRoles = async ( + getService: CommonFtrProviderContext['getService'], + usersToCreate: User[] = users, + rolesToCreate: Role[] = roles +) => { + const security = getService('security'); + + const createRole = async ({ name, privileges }: Role) => { + return await security.role.create(name, privileges); + }; + + const createUser = async (user: User) => { + const userInfo = getUserInfo(user); + + return await security.user.create(user.username, { + password: user.password, + roles: user.roles, + full_name: userInfo.full_name, + email: userInfo.email, + }); + }; + + await Promise.all(rolesToCreate.map((role) => createRole(role))); + await Promise.all(usersToCreate.map((user) => createUser(user))); +}; + +export const deleteUsersAndRoles = async ( + getService: CommonFtrProviderContext['getService'], + usersToDelete: User[] = users, + rolesToDelete: Role[] = roles +) => { + const security = getService('security'); + + try { + await Promise.allSettled(usersToDelete.map((user) => security.user.delete(user.username))); + } catch (error) { + // ignore errors because if a migration is run it will delete the .kibana index which remove the spaces and users + } + + try { + await Promise.allSettled(rolesToDelete.map((role) => security.role.delete(role.name))); + } catch (error) { + // ignore errors because if a migration is run it will delete the .kibana index which remove the spaces and users + } +}; + +export const createUsers = async (getService: CommonFtrProviderContext['getService']) => { + await createUsersAndRoles(getService); +}; + +export const deleteUsers = async (getService: CommonFtrProviderContext['getService']) => { + await deleteUsersAndRoles(getService); +}; + +export const activateUserProfiles = async (getService: CommonFtrProviderContext['getService']) => { + const supertestWithoutAuth = getService('supertestWithoutAuth'); + + await loginUsers({ + supertest: supertestWithoutAuth, + users: [noIntegrationsUser], + }); +}; diff --git a/x-pack/test/automatic_import_api_integration/common/lib/authentication/roles.ts b/x-pack/test/automatic_import_api_integration/common/lib/authentication/roles.ts new file mode 100644 index 0000000000000..dbd561aba4a58 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/authentication/roles.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Role } from './types'; + +export const noIntegrationsPrivileges: Role = { + name: 'no_integrations_kibana_privileges', + privileges: { + elasticsearch: { + indices: [ + { + names: ['*'], + privileges: ['all'], + }, + ], + }, + kibana: [ + { + feature: { + fleetv2: ['read'], + fleet: ['all'], + }, + spaces: ['*'], + }, + ], + }, +}; + +export const onlyActions: Role = { + name: 'only_actions', + privileges: { + elasticsearch: { + indices: [ + { + names: ['*'], + privileges: ['all'], + }, + ], + }, + kibana: [ + { + feature: { + actions: ['all'], + actionsSimulators: ['all'], + }, + spaces: ['*'], + }, + ], + }, +}; + +export const roles = [noIntegrationsPrivileges, onlyActions]; diff --git a/x-pack/test/automatic_import_api_integration/common/lib/authentication/types.ts b/x-pack/test/automatic_import_api_integration/common/lib/authentication/types.ts new file mode 100644 index 0000000000000..3bf3629441f93 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/authentication/types.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface Space { + id: string; + namespace?: string; + name: string; + disabledFeatures: string[]; +} + +export interface User { + username: string; + password: string; + description?: string; + roles: string[]; +} + +export interface UserInfo { + username: string; + full_name: string; + email: string; +} + +interface FeaturesPrivileges { + [featureId: string]: string[]; +} + +interface ElasticsearchIndices { + names: string[]; + privileges: string[]; +} + +export interface ElasticSearchPrivilege { + cluster?: string[]; + indices?: ElasticsearchIndices[]; +} + +export interface KibanaPrivilege { + spaces: string[]; + base?: string[]; + feature?: FeaturesPrivileges; +} + +export interface Role { + name: string; + privileges: { + elasticsearch?: ElasticSearchPrivilege; + kibana?: KibanaPrivilege[]; + }; +} diff --git a/x-pack/test/automatic_import_api_integration/common/lib/authentication/users.ts b/x-pack/test/automatic_import_api_integration/common/lib/authentication/users.ts new file mode 100644 index 0000000000000..6fea381d1e145 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/lib/authentication/users.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { noIntegrationsPrivileges, onlyActions as onlyActionsRole } from './roles'; +import { User } from './types'; + +export const superUser: User = { + username: 'superuser', + password: 'superuser', + roles: ['superuser'], +}; + +export const noIntegrationsUser: User = { + username: 'no_integrations_user', + password: 'no_integrations_user', + roles: [noIntegrationsPrivileges.name], +}; + +export const onlyActions: User = { + username: 'only_actions', + password: 'only_actions', + roles: [onlyActionsRole.name], +}; + +export const users = [superUser, noIntegrationsUser, onlyActions]; diff --git a/x-pack/test/automatic_import_api_integration/common/services.ts b/x-pack/test/automatic_import_api_integration/common/services.ts new file mode 100644 index 0000000000000..7e415338c405f --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/common/services.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { services } from '../../api_integration/services'; diff --git a/x-pack/test/automatic_import_api_integration/security/config_basic.ts b/x-pack/test/automatic_import_api_integration/security/config_basic.ts new file mode 100644 index 0000000000000..ebda3390790e3 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/security/config_basic.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../common/config'; + +// eslint-disable-next-line import/no-default-export +export default createTestConfig('security', { + license: 'basic', + ssl: true, + testFiles: [require.resolve('./tests/basic')], + publicBaseUrl: true, +}); diff --git a/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/analyze_logs.ts b/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/analyze_logs.ts new file mode 100644 index 0000000000000..6fb3abf127747 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/analyze_logs.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { postAnalyzeLogs } from '../../../../common/lib/api/analyze_logs'; +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; +import { User } from '../../../../common/lib/authentication/types'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + + describe('Run analyze logs', () => { + it('should get 404 when trying to run analyze_logs with basic license', async () => { + return await postAnalyzeLogs({ + supertest, + req: { + packageName: 'some-package', + dataStreamName: 'some-data-stream', + connectorId: 'bedrock-connector', + packageTitle: 'packageTitle', + dataStreamTitle: 'dataStreamTitle', + logSamples: ['sample1', 'sample2'], + }, + auth: { + user: { username: 'elastic', password: 'elastic' } as User, + }, + }); + }); + }); +}; diff --git a/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/categorization.ts b/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/categorization.ts new file mode 100644 index 0000000000000..4d9a3a0853109 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/categorization.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { postCategorization } from '../../../../common/lib/api/categorization'; +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; +import { User } from '../../../../common/lib/authentication/types'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + + describe('Run categorization', () => { + it('should get 404 when trying to run categorization with basic license', async () => { + return await postCategorization({ + supertest, + req: { + packageName: 'some-package', + dataStreamName: 'some-data-stream', + rawSamples: ['sample1', 'sample2'], + samplesFormat: { + name: 'json', + }, + connectorId: 'bedrock-connector', + currentPipeline: { + processors: [], + }, + }, + auth: { + user: { username: 'elastic', password: 'elastic' } as User, + }, + }); + }); + }); +}; diff --git a/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/ecs.ts b/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/ecs.ts new file mode 100644 index 0000000000000..d24a72f796d44 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/ecs.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { postEcsMapping } from '../../../../common/lib/api/ecs'; +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; +import { User } from '../../../../common/lib/authentication/types'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + + describe('Run ecs_mapping', () => { + it('should get 404 when trying to run ecs_mapping with basic license', async () => { + return await postEcsMapping({ + supertest, + req: { + packageName: 'some-package', + dataStreamName: 'some-data-stream', + rawSamples: ['sample1', 'sample2'], + samplesFormat: { + name: 'json', + }, + connectorId: 'bedrock-connector', + }, + auth: { + user: { username: 'elastic', password: 'elastic' } as User, + }, + }); + }); + }); +}; diff --git a/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/related.ts b/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/related.ts new file mode 100644 index 0000000000000..6091b64f7c224 --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/related.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { postRelated } from '../../../../common/lib/api/related'; +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; +import { User } from '../../../../common/lib/authentication/types'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + + describe('Run related', () => { + it('should get 404 when trying to run related graph with basic license', async () => { + return await postRelated({ + supertest, + req: { + packageName: 'some-package', + dataStreamName: 'some-data-stream', + rawSamples: ['sample1', 'sample2'], + samplesFormat: { + name: 'json', + }, + connectorId: 'bedrock-connector', + currentPipeline: { + processors: [], + }, + }, + auth: { + user: { username: 'elastic', password: 'elastic' } as User, + }, + }); + }); + }); +}; diff --git a/x-pack/test/automatic_import_api_integration/security/tests/basic/index.ts b/x-pack/test/automatic_import_api_integration/security/tests/basic/index.ts new file mode 100644 index 0000000000000..7391c1e3ae9ed --- /dev/null +++ b/x-pack/test/automatic_import_api_integration/security/tests/basic/index.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { + createUsersAndRoles, + deleteUsersAndRoles, + activateUserProfiles, +} from '../../../common/lib/authentication'; + +// eslint-disable-next-line import/no-default-export +export default ({ loadTestFile, getService }: FtrProviderContext): void => { + describe('Automatic Import enabled: basic', function () { + before(async () => { + await createUsersAndRoles(getService); + // once a user profile is created the only way to remove it is to delete the user and roles, so best to activate + // before all the tests + await activateUserProfiles(getService); + }); + + after(async () => { + await deleteUsersAndRoles(getService); + }); + + // Basic + loadTestFile(require.resolve('./graphs/ecs')); + }); +}; diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 9db41aecbb612..350ac68698acc 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -188,5 +188,6 @@ "@kbn/ai-assistant-common", "@kbn/core-deprecations-common", "@kbn/usage-collection-plugin", + "@kbn/integration-assistant-plugin" ] } From 9df737384642b57dc059e854be1236721f1a9bb3 Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Tue, 3 Dec 2024 16:00:31 +0100 Subject: [PATCH 11/51] [Lens] Stabilize FTR test (#202476) ## Summary Fixes #200120 The navigation bar is not focused for some reason. Making it wait a bit seems to stabilize it. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../functional/apps/lens/group2/persistent_context.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/test/functional/apps/lens/group2/persistent_context.ts b/x-pack/test/functional/apps/lens/group2/persistent_context.ts index 6a6f56578882b..b1e78d0ccb34d 100644 --- a/x-pack/test/functional/apps/lens/group2/persistent_context.ts +++ b/x-pack/test/functional/apps/lens/group2/persistent_context.ts @@ -73,9 +73,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await visualize.gotoVisualizationLandingPage(); await listingTable.searchForItemWithName('lnsTableVis'); await lens.clickVisualizeListItemTitle('lnsTableVis'); - await navigationalSearch.focus(); - await navigationalSearch.searchFor('type:application lens'); - await navigationalSearch.clickOnOption(0); + await retry.try(async () => { + await navigationalSearch.focus(); + await navigationalSearch.searchFor('type:application lens'); + await navigationalSearch.clickOnOption(0); + }); await lens.waitForEmptyWorkspace(); await lens.switchToVisualization('lnsLegacyMetric'); await lens.dragFieldToWorkspace('@timestamp', 'legacyMtrVis'); From 1556eb1fa010f8de59c9a5147ae68239012d0f0e Mon Sep 17 00:00:00 2001 From: Tre Date: Tue, 3 Dec 2024 15:32:41 +0000 Subject: [PATCH 12/51] [SKIP ON MKI] `x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts` (#202655) ## Summary See details: https://github.com/elastic/kibana/issues/202641 --- .../observability/dataset_quality/degraded_field_flyout.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 4b28ba5e897dd..7d12db16850cc 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -54,7 +54,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const apmAppDatasetName = 'apm.app.tug'; const apmAppDataStreamName = `${type}-${apmAppDatasetName}-${defaultNamespace}`; - describe('Degraded fields flyout', () => { + describe('Degraded fields flyout', function () { + // see details: https://github.com/elastic/kibana/issues/202641 + this.tags(['failsOnMKI']); describe('degraded field flyout open-close', () => { before(async () => { await synthtrace.index([ From 75842556c33fa06011b974f9065c7a0be541ed43 Mon Sep 17 00:00:00 2001 From: Kate Sosedova <36230123+ek-so@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:00:40 +0100 Subject: [PATCH 13/51] Fix the text color for a navigation callout (#202509) ## Summary We currently have a problem with the text inside the callout for new navigation which asks for the feedback. The PR makes color default so that it correlates with the regular text color of EUI. [Slack conversation](https://elastic.slack.com/archives/C7QC1JV6F/p1733133123930929). ![image](https://github.com/user-attachments/assets/1f6f5e96-1f87-496b-9cc4-3b33beb0efd3) --- .../chrome/navigation/src/ui/components/feedback_btn.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared-ux/chrome/navigation/src/ui/components/feedback_btn.tsx b/packages/shared-ux/chrome/navigation/src/ui/components/feedback_btn.tsx index 182838d88b5d6..3dbd0b3a25f61 100644 --- a/packages/shared-ux/chrome/navigation/src/ui/components/feedback_btn.tsx +++ b/packages/shared-ux/chrome/navigation/src/ui/components/feedback_btn.tsx @@ -50,7 +50,7 @@ export const FeedbackBtn: FC = ({ solutionId }) => { onDismiss={onDismiss} data-test-subj="sideNavfeedbackCallout" > - + {i18n.translate('sharedUXPackages.chrome.sideNavigation.feedbackCallout.title', { defaultMessage: `How's the navigation working for you? Missing anything?`, })} From 52b85f54bd655c2c6564aaaded252f03c67f2e56 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Tue, 3 Dec 2024 17:13:00 +0100 Subject: [PATCH 14/51] [Synthetics] Revert project payload limit (#202733) ## Summary This went into code by mistake https://github.com/elastic/kibana/pull/202467/files#r1867607838 !! --------- Co-authored-by: Dominique Clarke --- .../server/routes/monitor_cruds/add_monitor_project.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor_project.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor_project.ts index bc55c950a2e81..db8fbdd661f76 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor_project.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor_project.ts @@ -14,7 +14,7 @@ import { ProjectMonitor } from '../../../common/runtime_types'; import { SYNTHETICS_API_URLS } from '../../../common/constants'; import { ProjectMonitorFormatter } from '../../synthetics_service/project_monitor/project_monitor_formatter'; -const MAX_PAYLOAD_SIZE = 1048576 * 100; // 20MiB +const MAX_PAYLOAD_SIZE = 1048576 * 50; // 50MiB export const addSyntheticsProjectMonitorRoute: SyntheticsRestApiRouteFactory = () => ({ method: 'PUT', From 56f2e2258ae7c43ef64663c33e7eb90b1c50d9b0 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 3 Dec 2024 17:16:14 +0100 Subject: [PATCH 15/51] =?UTF-8?q?=F0=9F=8C=8A=20Adjust=20codeowners=20for?= =?UTF-8?q?=20streams=20code=20(#202367)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pending on adding the team to the repo --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 4 ++-- x-pack/plugins/streams/kibana.jsonc | 2 +- x-pack/plugins/streams_app/kibana.jsonc | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 51ab02111debd..b2c49732f70d3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -981,8 +981,8 @@ x-pack/plugins/snapshot_restore @elastic/kibana-management x-pack/plugins/spaces @elastic/kibana-security x-pack/plugins/stack_alerts @elastic/response-ops x-pack/plugins/stack_connectors @elastic/response-ops -x-pack/plugins/streams @simianhacker @flash1293 @dgieselaar -x-pack/plugins/streams_app @simianhacker @flash1293 @dgieselaar +x-pack/plugins/streams @elastic/streams-program-team +x-pack/plugins/streams_app @elastic/streams-program-team x-pack/plugins/task_manager @elastic/response-ops x-pack/plugins/telemetry_collection_xpack @elastic/kibana-core x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations diff --git a/x-pack/plugins/streams/kibana.jsonc b/x-pack/plugins/streams/kibana.jsonc index 06c37ed245cf1..b9ce6ef68e27e 100644 --- a/x-pack/plugins/streams/kibana.jsonc +++ b/x-pack/plugins/streams/kibana.jsonc @@ -1,7 +1,7 @@ { "type": "plugin", "id": "@kbn/streams-plugin", - "owner": "@simianhacker @flash1293 @dgieselaar", + "owner": "@elastic/streams-program-team", "description": "A manager for Streams", "group": "observability", "visibility": "private", diff --git a/x-pack/plugins/streams_app/kibana.jsonc b/x-pack/plugins/streams_app/kibana.jsonc index 16666084c53e5..c1480aba906de 100644 --- a/x-pack/plugins/streams_app/kibana.jsonc +++ b/x-pack/plugins/streams_app/kibana.jsonc @@ -1,7 +1,7 @@ { "type": "plugin", "id": "@kbn/streams-app-plugin", - "owner": "@simianhacker @flash1293 @dgieselaar", + "owner": "@elastic/streams-program-team", "group": "observability", "visibility": "private", "plugin": { From 230d6617ab7eff23eb1242630d2690b9cb68d422 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Tue, 3 Dec 2024 18:12:10 +0100 Subject: [PATCH 16/51] [Fleet] fix schema validation to allow undefined/null (#202732) ## Summary Fix a few issues encountered with schema validation. One of them reported here: https://discuss.elastic.co/t/fleet-error-updating-policy-settings/371332 The other encountered locally when testing upgrades: ``` "Failed output validation: [request body.items.0.upgrade_details]: expected a plain object value, but found [null] instead." ``` --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- oas_docs/bundle.json | 27 ++------------ oas_docs/bundle.serverless.json | 27 ++------------ oas_docs/output/kibana.serverless.yaml | 19 ++-------- oas_docs/output/kibana.yaml | 19 ++-------- .../fleet/common/types/models/agent_policy.ts | 2 +- .../fleet/server/types/models/agent_policy.ts | 2 +- .../fleet/server/types/rest_spec/agent.ts | 35 ++++++++++--------- 7 files changed, 33 insertions(+), 98 deletions(-) diff --git a/oas_docs/bundle.json b/oas_docs/bundle.json index b96aaa3a3ce00..0244960e1f55f 100644 --- a/oas_docs/bundle.json +++ b/oas_docs/bundle.json @@ -6547,9 +6547,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -7316,9 +7313,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -7577,9 +7571,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -8376,9 +8367,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -9436,9 +9424,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -10204,9 +10189,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -10465,9 +10447,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -11265,9 +11244,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -13278,6 +13254,7 @@ }, "upgrade_details": { "additionalProperties": false, + "nullable": true, "properties": { "action_id": { "type": "string" @@ -15470,6 +15447,7 @@ }, "upgrade_details": { "additionalProperties": false, + "nullable": true, "properties": { "action_id": { "type": "string" @@ -15958,6 +15936,7 @@ }, "upgrade_details": { "additionalProperties": false, + "nullable": true, "properties": { "action_id": { "type": "string" diff --git a/oas_docs/bundle.serverless.json b/oas_docs/bundle.serverless.json index 9115670ecb313..acfe609ea0c2f 100644 --- a/oas_docs/bundle.serverless.json +++ b/oas_docs/bundle.serverless.json @@ -6547,9 +6547,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -7316,9 +7313,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -7577,9 +7571,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -8376,9 +8367,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -9436,9 +9424,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -10204,9 +10189,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -10465,9 +10447,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -11265,9 +11244,6 @@ "type": "number" } }, - "required": [ - "enabled" - ], "type": "object" }, "monitoring_output_id": { @@ -13278,6 +13254,7 @@ }, "upgrade_details": { "additionalProperties": false, + "nullable": true, "properties": { "action_id": { "type": "string" @@ -15470,6 +15447,7 @@ }, "upgrade_details": { "additionalProperties": false, + "nullable": true, "properties": { "action_id": { "type": "string" @@ -15958,6 +15936,7 @@ }, "upgrade_details": { "additionalProperties": false, + "nullable": true, "properties": { "action_id": { "type": "string" diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index b605ab09de62f..142c0742614fb 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -9628,8 +9628,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -10160,8 +10158,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -10342,8 +10338,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -10896,8 +10890,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -11430,8 +11422,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -11961,8 +11951,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -12143,8 +12131,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -12697,8 +12683,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -14248,6 +14232,7 @@ paths: type: array upgrade_details: additionalProperties: false + nullable: true type: object properties: action_id: @@ -14715,6 +14700,7 @@ paths: type: array upgrade_details: additionalProperties: false + nullable: true type: object properties: action_id: @@ -15059,6 +15045,7 @@ paths: type: array upgrade_details: additionalProperties: false + nullable: true type: object properties: action_id: diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index b1f2186262365..0abbcc4b4a102 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -12489,8 +12489,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -13020,8 +13018,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -13202,8 +13198,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -13755,8 +13749,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -14288,8 +14280,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -14818,8 +14808,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -15000,8 +14988,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -15553,8 +15539,6 @@ paths: maximum: 65353 minimum: 0 type: number - required: - - enabled monitoring_output_id: nullable: true type: string @@ -17096,6 +17080,7 @@ paths: type: array upgrade_details: additionalProperties: false + nullable: true type: object properties: action_id: @@ -17560,6 +17545,7 @@ paths: type: array upgrade_details: additionalProperties: false + nullable: true type: object properties: action_id: @@ -17903,6 +17889,7 @@ paths: type: array upgrade_details: additionalProperties: false + nullable: true type: object properties: action_id: diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts index fb19953a1f731..0d180a0f4935a 100644 --- a/x-pack/plugins/fleet/common/types/models/agent_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/agent_policy.ts @@ -46,7 +46,7 @@ export interface NewAgentPolicy { global_data_tags?: GlobalDataTag[]; monitoring_pprof_enabled?: boolean; monitoring_http?: { - enabled: boolean; + enabled?: boolean; host?: string; port?: number; buffer?: { diff --git a/x-pack/plugins/fleet/server/types/models/agent_policy.ts b/x-pack/plugins/fleet/server/types/models/agent_policy.ts index 4f131d00bdf38..cc7dabe253ec1 100644 --- a/x-pack/plugins/fleet/server/types/models/agent_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/agent_policy.ts @@ -135,7 +135,7 @@ export const AgentPolicyBaseSchema = { monitoring_pprof_enabled: schema.maybe(schema.boolean()), monitoring_http: schema.maybe( schema.object({ - enabled: schema.boolean(), + enabled: schema.maybe(schema.boolean()), host: schema.maybe(schema.string({ defaultValue: 'localhost' })), port: schema.maybe(schema.number({ min: 0, max: 65353, defaultValue: 6791 })), buffer: schema.maybe(schema.object({ enabled: schema.boolean({ defaultValue: false }) })), diff --git a/x-pack/plugins/fleet/server/types/rest_spec/agent.ts b/x-pack/plugins/fleet/server/types/rest_spec/agent.ts index a5fb3e9e034ed..9df9b73b3f49b 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/agent.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/agent.ts @@ -138,22 +138,25 @@ export const AgentResponseSchema = schema.object({ upgraded_at: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), upgrade_started_at: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), upgrade_details: schema.maybe( - schema.object({ - target_version: schema.string(), - action_id: schema.string(), - state: AgentUpgradeStateTypeSchema, - metadata: schema.maybe( - schema.object({ - scheduled_at: schema.maybe(schema.string()), - download_percent: schema.maybe(schema.number()), - download_rate: schema.maybe(schema.number()), - failed_state: schema.maybe(AgentUpgradeStateTypeSchema), - error_msg: schema.maybe(schema.string()), - retry_error_msg: schema.maybe(schema.string()), - retry_until: schema.maybe(schema.string()), - }) - ), - }) + schema.oneOf([ + schema.literal(null), + schema.object({ + target_version: schema.string(), + action_id: schema.string(), + state: AgentUpgradeStateTypeSchema, + metadata: schema.maybe( + schema.object({ + scheduled_at: schema.maybe(schema.string()), + download_percent: schema.maybe(schema.number()), + download_rate: schema.maybe(schema.number()), + failed_state: schema.maybe(AgentUpgradeStateTypeSchema), + error_msg: schema.maybe(schema.string()), + retry_error_msg: schema.maybe(schema.string()), + retry_until: schema.maybe(schema.string()), + }) + ), + }), + ]) ), access_api_key_id: schema.maybe(schema.string()), default_api_key: schema.maybe(schema.string()), From b61ad41284381a6ac644d72e04928b6b79ae25c7 Mon Sep 17 00:00:00 2001 From: Marco Vettorello Date: Tue, 3 Dec 2024 18:17:49 +0100 Subject: [PATCH 17/51] [deps] Replace faker with @faker-js (#201105) ## Summary The `faker` library is[ not maintained anymore](https://fakerjs.dev/about/announcements/2022-01-14.html#i-heard-something-happened-what-s-the-tldr) and is replaced by a community fork `@faker-js`. This PR migrates all the usages of faker to the new library, trying to use the same methods (even if they have slight differences in results like `faker.random.number()` has a max of 99999 where instead `faker.number.int()` have a MAX_SAFE_INTEGER as max). --- package.json | 2 - .../src/calculate_width_from_entries.test.ts | 16 +++-- packages/kbn-dom-drag-drop/src/test_utils.tsx | 4 +- .../field_picker/field_picker.test.tsx | 6 +- .../public/components/metric_vis.test.tsx | 60 +++++++++---------- .../public/utils/filter_helpers.test.ts | 20 +++---- .../lens/public/app_plugin/app.test.tsx | 8 +-- .../public/app_plugin/app_helpers.test.ts | 2 +- .../app_plugin/save_modal_container.test.tsx | 8 +-- .../definitions/percentile.test.tsx | 4 +- .../chart_switch/chart_switch.test.tsx | 2 +- .../config_panel/layer_header.test.tsx | 2 +- .../config_panel/layer_panel.test.tsx | 4 +- .../workspace_panel/workspace_panel.test.tsx | 2 +- .../workspace_panel_wrapper.test.tsx | 4 +- .../color_telemetry_helpers.test.ts | 4 +- .../lens/public/mocks/visualization_mock.tsx | 2 +- .../expressions/on_event.test.ts | 8 +-- .../initializers/initialize_actions.test.ts | 4 +- .../initialize_dashboard_service.test.ts | 4 +- .../public/react_embeddable/mocks/index.tsx | 8 +-- .../state_management/load_initial.test.tsx | 2 +- .../datatable/components/table_basic.test.tsx | 12 ++-- .../metric/dimension_editor.test.tsx | 8 +-- .../infra/public/test_utils/entries.ts | 10 ++-- .../logs_shared/public/test_utils/entries.ts | 10 ++-- .../inference/connector.test.tsx | 18 ++++-- .../scripts/generate_indicators.js | 2 +- .../indicator_match_alert_suppression.ts | 2 +- .../utils/data_generator/get_timestamp.ts | 4 +- yarn.lock | 10 ---- 31 files changed, 122 insertions(+), 130 deletions(-) diff --git a/package.json b/package.json index eb82728877813..791295f9e8400 100644 --- a/package.json +++ b/package.json @@ -1572,7 +1572,6 @@ "@types/eslint": "^8.44.2", "@types/express": "^4.17.21", "@types/extract-zip": "^1.6.2", - "@types/faker": "^5.1.5", "@types/fetch-mock": "^7.3.1", "@types/file-saver": "^2.0.0", "@types/flot": "^0.0.31", @@ -1741,7 +1740,6 @@ "expect": "^29.7.0", "expose-loader": "^0.7.5", "express": "^4.21.1", - "faker": "^5.1.0", "fetch-mock": "^7.3.9", "file-loader": "^4.2.0", "find-cypress-specs": "^1.41.4", diff --git a/packages/kbn-calculate-width-from-char-count/src/calculate_width_from_entries.test.ts b/packages/kbn-calculate-width-from-char-count/src/calculate_width_from_entries.test.ts index 9924ea7b84a14..89abd5d8ce667 100644 --- a/packages/kbn-calculate-width-from-char-count/src/calculate_width_from_entries.test.ts +++ b/packages/kbn-calculate-width-from-char-count/src/calculate_width_from_entries.test.ts @@ -9,24 +9,22 @@ import { calculateWidthFromEntries } from './calculate_width_from_entries'; import { MAX_WIDTH } from './calculate_width_from_char_count'; -import faker from 'faker'; - -const generateLabel = (length: number) => faker.random.alpha({ count: length }); +import { faker } from '@faker-js/faker'; const generateObjectWithLabelOfLength = (length: number, propOverrides?: Record) => ({ - label: generateLabel(length), + label: faker.string.alpha(length), ...propOverrides, }); describe('calculateWidthFromEntries', () => { it('calculates width for array of strings', () => { - const shortLabels = [10, 20].map(generateLabel); + const shortLabels = [10, 20].map(faker.string.alpha); expect(calculateWidthFromEntries(shortLabels)).toBe(256); - const mediumLabels = [50, 55, 10, 20].map(generateLabel); + const mediumLabels = [50, 55, 10, 20].map(faker.string.alpha); expect(calculateWidthFromEntries(mediumLabels)).toBe(501); - const longLabels = [80, 90, 10].map(generateLabel); + const longLabels = [80, 90, 10].map(faker.string.alpha); expect(calculateWidthFromEntries(longLabels)).toBe(MAX_WIDTH); }); @@ -42,12 +40,12 @@ describe('calculateWidthFromEntries', () => { }); it('calculates width for array of objects for fallback keys', () => { const shortLabels = [10, 20].map((v) => - generateObjectWithLabelOfLength(v, { label: undefined, name: generateLabel(v) }) + generateObjectWithLabelOfLength(v, { label: undefined, name: faker.string.alpha(v) }) ); expect(calculateWidthFromEntries(shortLabels, ['id', 'label', 'name'])).toBe(256); const mediumLabels = [50, 55, 10, 20].map((v) => - generateObjectWithLabelOfLength(v, { label: undefined, name: generateLabel(v) }) + generateObjectWithLabelOfLength(v, { label: undefined, name: faker.string.alpha(v) }) ); expect(calculateWidthFromEntries(mediumLabels, ['id', 'label', 'name'])).toBe(501); }); diff --git a/packages/kbn-dom-drag-drop/src/test_utils.tsx b/packages/kbn-dom-drag-drop/src/test_utils.tsx index 033697353a43f..84996956e26c3 100644 --- a/packages/kbn-dom-drag-drop/src/test_utils.tsx +++ b/packages/kbn-dom-drag-drop/src/test_utils.tsx @@ -8,7 +8,7 @@ */ import React, { ReactElement } from 'react'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { RenderOptions, render } from '@testing-library/react'; import { DragContextState, RootDragDropProvider } from './providers'; @@ -22,7 +22,7 @@ export const dataTransfer = { }; export const generateDragDropValue = (label = faker.lorem.word()) => ({ - id: faker.random.uuid(), + id: faker.string.uuid(), humanData: { label, groupLabel: faker.lorem.word(), diff --git a/packages/kbn-visualization-ui-components/components/field_picker/field_picker.test.tsx b/packages/kbn-visualization-ui-components/components/field_picker/field_picker.test.tsx index 6cdcfaaa8d0ad..a19de192662c2 100644 --- a/packages/kbn-visualization-ui-components/components/field_picker/field_picker.test.tsx +++ b/packages/kbn-visualization-ui-components/components/field_picker/field_picker.test.tsx @@ -10,15 +10,15 @@ import React from 'react'; import { FieldPicker, FieldPickerProps } from './field_picker'; import { render, screen } from '@testing-library/react'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import userEvent from '@testing-library/user-event'; import { DataType, FieldOptionValue } from './types'; const generateFieldWithLabelOfLength = (length: number) => ({ - label: faker.random.alpha({ count: length }), + label: faker.string.alpha(length), value: { type: 'field' as const, - field: faker.random.alpha({ count: length }), + field: faker.string.alpha(length), dataType: 'date' as DataType, operationType: 'count', }, diff --git a/src/plugins/chart_expressions/expression_metric/public/components/metric_vis.test.tsx b/src/plugins/chart_expressions/expression_metric/public/components/metric_vis.test.tsx index fd0892ea12735..38d8e4d695c8d 100644 --- a/src/plugins/chart_expressions/expression_metric/public/components/metric_vis.test.tsx +++ b/src/plugins/chart_expressions/expression_metric/public/components/metric_vis.test.tsx @@ -27,7 +27,7 @@ import { CustomPaletteState } from '@kbn/charts-plugin/common/expressions/palett import { DimensionsVisParam, MetricVisParam } from '../../common'; import { euiThemeVars } from '@kbn/ui-theme'; import { DEFAULT_TRENDLINE_NAME } from '../../common/constants'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; const mockDeserialize = jest.fn(({ id }: { id: string }) => { const convertFn = (v: unknown) => `${id}-${v}`; @@ -825,47 +825,47 @@ describe('MetricVisComponent', function () { // Raw values here, not formatted const trends: Record = { Friday: [ - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, ], Wednesday: [ - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, ], Saturday: [ - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, ], Sunday: [ - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, ], Thursday: [ - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, ], __other__: [ - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, ], // this one shouldn't show up! [DEFAULT_TRENDLINE_NAME]: [ - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, - { x: faker.random.number(), y: faker.random.number() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, + { x: faker.number.int(), y: faker.number.int() }, ], }; @@ -1004,7 +1004,7 @@ describe('MetricVisComponent', function () { }); it('should convert null values to NaN', () => { - const metricId = faker.random.word(); + const metricId = faker.lorem.word(); const tableWNull: Datatable = { type: 'datatable', diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/filter_helpers.test.ts b/src/plugins/chart_expressions/expression_partition_vis/public/utils/filter_helpers.test.ts index babde416ea0a5..d9a6bb9d95631 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/filter_helpers.test.ts +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/filter_helpers.test.ts @@ -18,7 +18,7 @@ import { import { createMockBucketColumns, createMockVisData, createMockPieParams } from '../mocks'; import { consolidateMetricColumns } from '../../common/utils'; import { LayerValue } from '@elastic/charts'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; const bucketColumns = createMockBucketColumns(); const visData = createMockVisData(); @@ -186,26 +186,26 @@ describe('getFilterClickData', () => { const clickedLayers: LayerValue[] = [ { groupByRollup: 'circle', - value: faker.random.number(), - depth: faker.random.number(), + value: faker.number.int(), + depth: faker.number.int(), path: [], - sortIndex: faker.random.number(), + sortIndex: faker.number.int(), smAccessorValue: '', }, { groupByRollup: 'green', - value: faker.random.number(), - depth: faker.random.number(), + value: faker.number.int(), + depth: faker.number.int(), path: [], - sortIndex: faker.random.number(), + sortIndex: faker.number.int(), smAccessorValue: '', }, { groupByRollup: 'metric2', - value: faker.random.number(), - depth: faker.random.number(), + value: faker.number.int(), + depth: faker.number.int(), path: [], - sortIndex: faker.random.number(), + sortIndex: faker.number.int(), smAccessorValue: '', }, ]; diff --git a/x-pack/plugins/lens/public/app_plugin/app.test.tsx b/x-pack/plugins/lens/public/app_plugin/app.test.tsx index 73fb52bbe6683..c146352ede566 100644 --- a/x-pack/plugins/lens/public/app_plugin/app.test.tsx +++ b/x-pack/plugins/lens/public/app_plugin/app.test.tsx @@ -34,7 +34,7 @@ import { setState, LensAppState } from '../state_management'; import { coreMock } from '@kbn/core/public/mocks'; import { LensSerializedState } from '..'; import { createMockedField, createMockedIndexPattern } from '../datasources/form_based/mocks'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { VisualizeEditorContext } from '../types'; @@ -48,7 +48,7 @@ jest.mock('lodash', () => ({ debounce: (fn: unknown) => fn, })); -const defaultSavedObjectId: string = faker.random.uuid(); +const defaultSavedObjectId: string = faker.string.uuid(); const waitToLoad = async () => await act(async () => new Promise((resolve) => setTimeout(resolve, 0))); @@ -233,7 +233,7 @@ describe('Lens App', () => { }); describe('breadcrumbs', () => { - const breadcrumbDocSavedObjectId = faker.random.uuid(); + const breadcrumbDocSavedObjectId = faker.string.uuid(); const breadcrumbDoc = getLensDocumentMock({ savedObjectId: breadcrumbDocSavedObjectId, title: 'Daaaaaaadaumching!', @@ -601,7 +601,7 @@ describe('Lens App', () => { expect(lensStore.getState().lens.applyChangesCounter).toBe(1); }); it('adds to the recently accessed list on save', async () => { - const savedObjectId = faker.random.uuid(); + const savedObjectId = faker.string.uuid(); await save({ savedObjectId, prevSavedObjectId: 'prevId', comesFromDashboard: false }); expect(services.chrome.recentlyAccessed.add).toHaveBeenCalledWith( `/app/lens#/edit/${savedObjectId}`, diff --git a/x-pack/plugins/lens/public/app_plugin/app_helpers.test.ts b/x-pack/plugins/lens/public/app_plugin/app_helpers.test.ts index 7dc4e8cfda78c..773276710f48d 100644 --- a/x-pack/plugins/lens/public/app_plugin/app_helpers.test.ts +++ b/x-pack/plugins/lens/public/app_plugin/app_helpers.test.ts @@ -6,7 +6,7 @@ */ import { renderHook, act } from '@testing-library/react-hooks'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { UseNavigateBackToAppProps, useNavigateBackToApp } from './app_helpers'; import { defaultDoc, makeDefaultServices } from '../mocks/services_mock'; import { cloneDeep } from 'lodash'; diff --git a/x-pack/plugins/lens/public/app_plugin/save_modal_container.test.tsx b/x-pack/plugins/lens/public/app_plugin/save_modal_container.test.tsx index 987b320b3abf1..5d9c833174ed5 100644 --- a/x-pack/plugins/lens/public/app_plugin/save_modal_container.test.tsx +++ b/x-pack/plugins/lens/public/app_plugin/save_modal_container.test.tsx @@ -8,7 +8,7 @@ import { SaveProps } from './app'; import { type SaveVisualizationProps, runSaveLensVisualization } from './save_modal_container'; import { defaultDoc, makeDefaultServices } from '../mocks'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { makeAttributeService } from '../mocks/services_mock'; jest.mock('../persistence/saved_objects_utils/check_for_duplicate_title', () => ({ @@ -241,7 +241,7 @@ describe('runSaveLensVisualization', () => { }); // save the current document as a new by-value copy and add it to a dashboard it('should save as a new by-value copy and redirect to the dashboard', async () => { - const dashboardId = faker.random.uuid(); + const dashboardId = faker.string.uuid(); const { props, saveProps, options, redirectToDashboardFn, saveToLibraryFn, toasts } = getDefaultArgs( { @@ -271,7 +271,7 @@ describe('runSaveLensVisualization', () => { // save the current document as a new by-ref copy and add it to a dashboard it('should save as a new by-ref copy and redirect to the dashboard', async () => { - const dashboardId = faker.random.uuid(); + const dashboardId = faker.string.uuid(); const { props, saveProps, options, redirectToDashboardFn, saveToLibraryFn, toasts } = getDefaultArgs( { @@ -376,7 +376,7 @@ describe('runSaveLensVisualization', () => { // simulate a new save const attributeServiceMock = makeAttributeService({ ...defaultDoc, - savedObjectId: faker.random.uuid(), + savedObjectId: faker.string.uuid(), }); const { props, saveProps, options, saveToLibraryFn, cleanupEditor } = getDefaultArgs( diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.test.tsx index a140ca22db2b5..b8d3af522c455 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.test.tsx @@ -12,7 +12,7 @@ import { unifiedSearchPluginMock } from '@kbn/unified-search-plugin/public/mocks import { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; import { dataPluginMock } from '@kbn/data-plugin/public/mocks'; import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { createMockedIndexPattern } from '../../mocks'; @@ -402,7 +402,7 @@ describe('percentile', () => { it('should update order-by references for any terms columns', () => { const field1 = 'foo'; const field2 = 'bar'; - const percentile = faker.random.number(100); + const percentile = faker.number.int(100); const aggs = [ makeEsAggBuilder('aggTerms', { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch/chart_switch.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch/chart_switch.test.tsx index 6795cd8caaa83..5686d94f52d49 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch/chart_switch.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch/chart_switch.test.tsx @@ -21,7 +21,7 @@ import { DatasourcePublicAPI, SuggestionRequest, DatasourceSuggestion } from '.. import { ChartSwitchProps } from './chart_switch'; import { ChartSwitchPopover } from './chart_switch_popover'; import { LensAppState, applyChanges } from '../../../../state_management'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; const mockFrame = (layers: string[]) => ({ ...createMockFramePublicAPI(), diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_header.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_header.test.tsx index 6f0edd6cbcd16..bcd21c71312ce 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_header.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_header.test.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { screen } from '@testing-library/react'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { createMockDatasource, createMockFramePublicAPI, diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx index 9c3457dacc6d4..9cfb1597ffdd4 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx @@ -12,7 +12,7 @@ import { ChildDragDropProvider, Droppable, Draggable } from '@kbn/dom-drag-drop' import { FramePublicAPI, Visualization, VisualizationConfigProps } from '../../../types'; import { LayerPanel } from './layer_panel'; import { coreMock } from '@kbn/core/public/mocks'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { generateId } from '../../../id_generator'; import { createMockVisualization, @@ -144,7 +144,7 @@ describe('LayerPanel', () => { }; beforeEach(() => { - mockVisualization = createMockVisualization(faker.random.alphaNumeric()); + mockVisualization = createMockVisualization(faker.string.alphanumeric()); mockVisualization.getLayerIds.mockReturnValue(['first']); mockDatasource = createMockDatasource(); }); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx index 42abcdf8fe56c..e44e528e5a33c 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx @@ -39,7 +39,7 @@ import { getLensInspectorService } from '../../../lens_inspector_service'; import { inspectorPluginMock } from '@kbn/inspector-plugin/public/mocks'; import { disableAutoApply, enableAutoApply } from '../../../state_management/lens_slice'; import { Ast, toExpression } from '@kbn/interpreter'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; const defaultPermissions: Record>> = { navLinks: { management: true }, diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx index 8694cc7c27dcf..c5cabaa053c4b 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx @@ -18,7 +18,7 @@ import { updateVisualizationState, LensAppState } from '../../../state_managemen import { setChangesApplied } from '../../../state_management/lens_slice'; import { LensInspector } from '../../../lens_inspector_service'; import { screen } from '@testing-library/react'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { SettingsMenu } from '../../../app_plugin/settings_menu'; describe('workspace_panel_wrapper', () => { @@ -98,7 +98,7 @@ describe('workspace_panel_wrapper', () => { }); it('should render its children', async () => { - const customElementText = faker.random.word(); + const customElementText = faker.word.words(); renderWorkspacePanelWrapper({ children: {customElementText} }); expect(screen.getByText(customElementText)).toBeInTheDocument(); }); diff --git a/x-pack/plugins/lens/public/lens_ui_telemetry/color_telemetry_helpers.test.ts b/x-pack/plugins/lens/public/lens_ui_telemetry/color_telemetry_helpers.test.ts index 47eba8b0252ef..8719c54f6f1dd 100644 --- a/x-pack/plugins/lens/public/lens_ui_telemetry/color_telemetry_helpers.test.ts +++ b/x-pack/plugins/lens/public/lens_ui_telemetry/color_telemetry_helpers.test.ts @@ -13,7 +13,7 @@ import { DEFAULT_COLOR_MAPPING_CONFIG, DEFAULT_OTHER_ASSIGNMENT_INDEX, } from '@kbn/coloring'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; const exampleAssignment = ( valuesCount = 1, @@ -35,7 +35,7 @@ const exampleAssignment = ( return { rule: { type: 'matchExactly', - values: Array.from({ length: valuesCount }, () => faker.random.alpha()), + values: Array.from({ length: valuesCount }, () => faker.string.alpha()), }, color, touched: false, diff --git a/x-pack/plugins/lens/public/mocks/visualization_mock.tsx b/x-pack/plugins/lens/public/mocks/visualization_mock.tsx index f0513e66de0c5..9810b916d5ea6 100644 --- a/x-pack/plugins/lens/public/mocks/visualization_mock.tsx +++ b/x-pack/plugins/lens/public/mocks/visualization_mock.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { LayerTypes } from '@kbn/expression-xy-plugin/public'; import { toExpression } from '@kbn/interpreter'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { Visualization, VisualizationMap } from '../types'; export function createMockVisualization( diff --git a/x-pack/plugins/lens/public/react_embeddable/expressions/on_event.test.ts b/x-pack/plugins/lens/public/react_embeddable/expressions/on_event.test.ts index dfddfe84b57cc..aba94db03de88 100644 --- a/x-pack/plugins/lens/public/react_embeddable/expressions/on_event.test.ts +++ b/x-pack/plugins/lens/public/react_embeddable/expressions/on_event.test.ts @@ -9,7 +9,7 @@ import { ExpressionRendererEvent } from '@kbn/expressions-plugin/public'; import { getLensApiMock, getLensRuntimeStateMock, makeEmbeddableServices } from '../mocks'; import { LensApi, LensEmbeddableStartServices, LensPublicCallbacks } from '../types'; import { prepareEventHandler } from './on_event'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { LENS_EDIT_PAGESIZE_ACTION, LENS_EDIT_RESIZE_ACTION, @@ -109,7 +109,7 @@ describe('Embeddable interaction event handlers', () => { const event = { name: 'filter', data: { - data: [{ value: faker.random.number(), row: 1, column: 'test', table: getTable() }], + data: [{ value: faker.number.int(), row: 1, column: 'test', table: getTable() }], }, }; const { callbacks } = await submitEvent(event); @@ -119,7 +119,7 @@ describe('Embeddable interaction event handlers', () => { const event = { name: 'filter', data: { - data: [{ value: faker.random.number(), row: 1, column: 'test', table: getTable() }], + data: [{ value: faker.number.int(), row: 1, column: 'test', table: getTable() }], }, }; const { getTrigger } = await submitEvent(event, true); @@ -171,7 +171,7 @@ describe('Embeddable interaction event handlers', () => { await reSubmit({ name: 'filter', data: { - data: [{ value: faker.random.number(), row: 1, column: 'test', table: getTable() }], + data: [{ value: faker.number.int(), row: 1, column: 'test', table: getTable() }], }, }); diff --git a/x-pack/plugins/lens/public/react_embeddable/initializers/initialize_actions.test.ts b/x-pack/plugins/lens/public/react_embeddable/initializers/initialize_actions.test.ts index a4f84c329bd3c..016bc0e87f11d 100644 --- a/x-pack/plugins/lens/public/react_embeddable/initializers/initialize_actions.test.ts +++ b/x-pack/plugins/lens/public/react_embeddable/initializers/initialize_actions.test.ts @@ -6,7 +6,7 @@ */ import { pick } from 'lodash'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import type { LensRuntimeState, VisualizationContext } from '../types'; import { initializeActionApi } from './initialize_actions'; import { @@ -42,7 +42,7 @@ function setupActionsApi( visOverrides: { id: 'lnsXY' }, dataOverrides: { id: 'form_based' }, }); - const uuid = faker.random.uuid(); + const uuid = faker.string.uuid(); const runtimeState = getLensRuntimeStateMock(stateOverrides); const apiMock = getLensApiMock(); diff --git a/x-pack/plugins/lens/public/react_embeddable/initializers/initialize_dashboard_service.test.ts b/x-pack/plugins/lens/public/react_embeddable/initializers/initialize_dashboard_service.test.ts index 2a0c469b3bbfb..af68f887bff54 100644 --- a/x-pack/plugins/lens/public/react_embeddable/initializers/initialize_dashboard_service.test.ts +++ b/x-pack/plugins/lens/public/react_embeddable/initializers/initialize_dashboard_service.test.ts @@ -9,7 +9,7 @@ import type { LensRuntimeState } from '../types'; import { getLensRuntimeStateMock, getLensInternalApiMock, makeEmbeddableServices } from '../mocks'; import { initializeStateManagement } from './initialize_state_management'; import { initializeDashboardServices } from './initialize_dashboard_services'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { createEmptyLensState } from '../helper'; function setupDashboardServicesApi(runtimeOverrides?: Partial) { @@ -30,7 +30,7 @@ function setupDashboardServicesApi(runtimeOverrides?: Partial) describe('Transformation API', () => { it("should not save to library if there's already a saveObjectId", async () => { - const api = setupDashboardServicesApi({ savedObjectId: faker.random.uuid() }); + const api = setupDashboardServicesApi({ savedObjectId: faker.string.uuid() }); expect(await api.canLinkToLibrary()).toBe(false); }); diff --git a/x-pack/plugins/lens/public/react_embeddable/mocks/index.tsx b/x-pack/plugins/lens/public/react_embeddable/mocks/index.tsx index ddcd5e6089592..ca8f489a711aa 100644 --- a/x-pack/plugins/lens/public/react_embeddable/mocks/index.tsx +++ b/x-pack/plugins/lens/public/react_embeddable/mocks/index.tsx @@ -8,7 +8,7 @@ import { BehaviorSubject, Subject } from 'rxjs'; import deepMerge from 'deepmerge'; import React from 'react'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { Query, Filter, AggregateQuery, TimeRange } from '@kbn/es-query'; import { PhaseEvent, ViewMode } from '@kbn/presentation-publishing'; import { DataView } from '@kbn/data-views-plugin/common'; @@ -49,7 +49,7 @@ import { const LensApiMock: LensApi = { // Static props type: DOC_TYPE, - uuid: faker.random.uuid(), + uuid: faker.string.uuid(), // Shared Embeddable Observables panelTitle: new BehaviorSubject(faker.lorem.words()), hidePanelTitle: new BehaviorSubject(false), @@ -94,7 +94,7 @@ const LensApiMock: LensApi = { setPanelTitle: jest.fn(), setHidePanelTitle: jest.fn(), phase$: new BehaviorSubject({ - id: faker.random.uuid(), + id: faker.string.uuid(), status: 'rendered', timeToEvent: 1000, }), @@ -138,7 +138,7 @@ export function getLensApiMock(overrides: Partial = {}) { export function getLensSerializedStateMock(overrides: Partial = {}) { return { - savedObjectId: faker.random.uuid(), + savedObjectId: faker.string.uuid(), ...LensSerializedStateMock, ...overrides, }; diff --git a/x-pack/plugins/lens/public/state_management/load_initial.test.tsx b/x-pack/plugins/lens/public/state_management/load_initial.test.tsx index 1514a508b8781..0a47af299d136 100644 --- a/x-pack/plugins/lens/public/state_management/load_initial.test.tsx +++ b/x-pack/plugins/lens/public/state_management/load_initial.test.tsx @@ -17,7 +17,7 @@ import { Location, History } from 'history'; import { act } from 'react-dom/test-utils'; import { InitialAppState, loadInitial } from './lens_slice'; import { Filter } from '@kbn/es-query'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { DOC_TYPE } from '../../common/constants'; const history = { diff --git a/x-pack/plugins/lens/public/visualizations/datatable/components/table_basic.test.tsx b/x-pack/plugins/lens/public/visualizations/datatable/components/table_basic.test.tsx index 14b3796fbd145..992e79681d415 100644 --- a/x-pack/plugins/lens/public/visualizations/datatable/components/table_basic.test.tsx +++ b/x-pack/plugins/lens/public/visualizations/datatable/components/table_basic.test.tsx @@ -9,7 +9,7 @@ import React from 'react'; import { fireEvent, render, screen, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { I18nProvider } from '@kbn/i18n-react'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { act } from 'react-dom/test-utils'; import { IFieldFormat } from '@kbn/field-formats-plugin/common'; import { coreMock } from '@kbn/core/public/mocks'; @@ -504,7 +504,7 @@ describe('DatatableComponent', () => { data.rows = new Array(rowNumbers).fill({ a: 'shoes', b: 1588024800000, - c: faker.random.number(), + c: faker.number.int(), }); args.pageSize = pageSize; @@ -537,7 +537,7 @@ describe('DatatableComponent', () => { data.rows = new Array(rowNumbers).fill({ a: 'shoes', b: 1588024800000, - c: faker.random.number(), + c: faker.number.int(), }); args.pageSize = pageSize; @@ -558,7 +558,7 @@ describe('DatatableComponent', () => { data.rows = new Array(20).fill({ a: 'shoes', b: 1588024800000, - c: faker.random.number(), + c: faker.number.int(), }); renderDatatableComponent({ args, @@ -584,7 +584,7 @@ describe('DatatableComponent', () => { data.rows = new Array(rowNumbers).fill({ a: 'shoes', b: 1588024800000, - c: faker.random.number(), + c: faker.number.int(), }); args.pageSize = pageSize; @@ -624,7 +624,7 @@ describe('DatatableComponent', () => { data.rows = new Array(rowNumbers).fill({ a: 'shoes', b: 1588024800000, - c: faker.random.number(), + c: faker.number.int(), }); args.pageSize = pageSize; diff --git a/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.test.tsx b/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.test.tsx index 73f07d66bcb8b..7a143d3c9ba8a 100644 --- a/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.test.tsx +++ b/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.test.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { render, screen, waitFor } from '@testing-library/react'; -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import userEvent from '@testing-library/user-event'; import { chartPluginMock } from '@kbn/charts-plugin/public/mocks'; import { euiLightVars } from '@kbn/ui-theme'; @@ -77,7 +77,7 @@ describe('dimension editor', () => { id: 'first', rows: Array(3).fill({ 'metric-col-id': faker.lorem.word(3), - 'max-col-id': faker.random.number(), + 'max-col-id': faker.number.int(), }), }, ]), @@ -106,8 +106,8 @@ describe('dimension editor', () => { { id: 'first', rows: Array(3).fill({ - 'metric-col-id': faker.random.number(), - 'secondary-metric-col-id': faker.random.number(), + 'metric-col-id': faker.number.int(), + 'secondary-metric-col-id': faker.number.int(), }), }, ]), diff --git a/x-pack/plugins/observability_solution/infra/public/test_utils/entries.ts b/x-pack/plugins/observability_solution/infra/public/test_utils/entries.ts index 0b04c90ee633d..7042a6538dceb 100644 --- a/x-pack/plugins/observability_solution/infra/public/test_utils/entries.ts +++ b/x-pack/plugins/observability_solution/infra/public/test_utils/entries.ts @@ -5,7 +5,7 @@ * 2.0. */ -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { LogEntry, LogViewColumnConfiguration } from '@kbn/logs-shared-plugin/common'; export function generateFakeEntries( @@ -51,15 +51,15 @@ export function generateFakeEntries( function fakeColumnValue(field: string): string { switch (field) { case 'message': - return faker.fake( - '{{internet.ip}} - [{{date.past}}] "GET {{internet.url}} HTTP/1.1" 200 {{random.number}} "-" "{{internet.userAgent}}"' + return faker.helpers.fake( + '{{internet.ip}} - [{{date.past}}] "GET {{internet.url}} HTTP/1.1" 200 {{number.int}} "-" "{{internet.userAgent}}"' ); case 'event.dataset': - return faker.fake('{{hacker.noun}}.{{hacker.noun}}'); + return faker.helpers.fake('{{hacker.noun}}.{{hacker.noun}}'); case 'log.file.path': return faker.system.filePath(); case 'log.level': - return faker.random.arrayElement(['debug', 'info', 'warn', 'error']); + return faker.helpers.arrayElement(['debug', 'info', 'warn', 'error']); case 'host.name': return faker.hacker.noun(); default: diff --git a/x-pack/plugins/observability_solution/logs_shared/public/test_utils/entries.ts b/x-pack/plugins/observability_solution/logs_shared/public/test_utils/entries.ts index 5277f49b1175e..dc6a1526bba0c 100644 --- a/x-pack/plugins/observability_solution/logs_shared/public/test_utils/entries.ts +++ b/x-pack/plugins/observability_solution/logs_shared/public/test_utils/entries.ts @@ -5,7 +5,7 @@ * 2.0. */ -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import { LogEntry } from '../../common/log_entry'; import { LogViewColumnConfiguration } from '../../common/log_views'; @@ -60,15 +60,15 @@ export function generateFakeEntries( function fakeColumnValue(field: string): string { switch (field) { case 'message': - return faker.fake( - '{{internet.ip}} - [{{date.past}}] "GET {{internet.url}} HTTP/1.1" 200 {{random.number}} "-" "{{internet.userAgent}}"' + return faker.helpers.fake( + '{{internet.ip}} - [{{date.past}}] "GET {{internet.url}} HTTP/1.1" 200 {{number.int}} "-" "{{internet.userAgent}}"' ); case 'event.dataset': - return faker.fake('{{hacker.noun}}.{{hacker.noun}}'); + return faker.helpers.fake('{{hacker.noun}}.{{hacker.noun}}'); case 'log.file.path': return faker.system.filePath(); case 'log.level': - return faker.random.arrayElement(['debug', 'info', 'warn', 'error']); + return faker.helpers.arrayElement(['debug', 'info', 'warn', 'error']); case 'host.name': return faker.hacker.noun(); default: diff --git a/x-pack/plugins/stack_connectors/public/connector_types/inference/connector.test.tsx b/x-pack/plugins/stack_connectors/public/connector_types/inference/connector.test.tsx index d445504011b5f..4b4beb16d798e 100644 --- a/x-pack/plugins/stack_connectors/public/connector_types/inference/connector.test.tsx +++ b/x-pack/plugins/stack_connectors/public/connector_types/inference/connector.test.tsx @@ -25,13 +25,19 @@ jest.mock('@kbn/triggers-actions-ui-plugin/public/common/lib/kibana', () => ({ })), })); -jest.mock('@faker-js/faker', () => ({ - faker: { - string: { - alpha: jest.fn().mockReturnValue('123'), +jest.mock('@faker-js/faker', () => { + const originalModule = jest.requireActual('@faker-js/faker'); + return { + ...originalModule, + faker: { + ...originalModule.faker, + string: { + ...originalModule.faker.string, + alpha: jest.fn().mockReturnValue('123'), + }, }, - }, -})); + }; +}); const mockProviders = useProviders as jest.Mock; diff --git a/x-pack/plugins/threat_intelligence/scripts/generate_indicators.js b/x-pack/plugins/threat_intelligence/scripts/generate_indicators.js index 0d5c170965fe7..daa869c76cf70 100644 --- a/x-pack/plugins/threat_intelligence/scripts/generate_indicators.js +++ b/x-pack/plugins/threat_intelligence/scripts/generate_indicators.js @@ -6,7 +6,7 @@ */ const { Client } = require('@elastic/elasticsearch'); -const faker = require('faker'); +const { faker } = require('@faker-js/faker'); const THREAT_INDEX = 'logs-ti'; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_alert_suppression.ts index 1acb416808081..6ede4121610e6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_alert_suppression.ts @@ -1250,7 +1250,7 @@ export default ({ getService }: FtrProviderContext) => { await indexGeneratedSourceDocuments({ docsCount: 60000, - interval: [firstTimestamp, '2020-10-28T05:35:50.000Z'], + interval: [firstTimestamp, '2020-10-28T05:45:50.000Z'], seed: (index, _, timestamp) => ({ id, '@timestamp': timestamp, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/data_generator/get_timestamp.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/data_generator/get_timestamp.ts index e828767a39650..eb0de446d09d2 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/data_generator/get_timestamp.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/data_generator/get_timestamp.ts @@ -4,12 +4,12 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import faker from 'faker'; +import { faker } from '@faker-js/faker'; import type { IndexingInterval } from './types'; export const getTimestamp = (interval?: IndexingInterval) => { if (interval) { - return faker.date.between(...interval).toISOString(); + return faker.date.between({ from: interval[0], to: interval[1] }).toISOString(); } return new Date().toISOString(); diff --git a/yarn.lock b/yarn.lock index 70875c1d31374..9a1bdc77bd750 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11493,11 +11493,6 @@ resolved "https://registry.yarnpkg.com/@types/extract-zip/-/extract-zip-1.6.2.tgz#5c7eb441c41136167a42b88b64051e6260c29e86" integrity sha1-XH60QcQRNhZ6QriLZAUeYmDCnoY= -"@types/faker@^5.1.5": - version "5.1.5" - resolved "https://registry.yarnpkg.com/@types/faker/-/faker-5.1.5.tgz#f14b015e0100232bb00c6dd7611505efb08709a0" - integrity sha512-2uEQFb7bsx68rqD4F8q95wZq6LTLOyexjv6BnvJogCO4jStkyc6IDEkODPQcWfovI6g6M3uPQ2/uD/oedJKkNw== - "@types/fetch-mock@^7.3.1": version "7.3.1" resolved "https://registry.yarnpkg.com/@types/fetch-mock/-/fetch-mock-7.3.1.tgz#df7421e8bcb351b430bfbfa5c52bb353826ac94f" @@ -18764,11 +18759,6 @@ extsprintf@1.3.0, extsprintf@^1.2.0: resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= -faker@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/faker/-/faker-5.1.0.tgz#e10fa1dec4502551aee0eb771617a7e7b94692e8" - integrity sha512-RrWKFSSA/aNLP0g3o2WW1Zez7/MnMr7xkiZmoCfAGZmdkDQZ6l2KtuXHN5XjdvpRjDl8+3vf+Rrtl06Z352+Mw== - fancy-log@^1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/fancy-log/-/fancy-log-1.3.3.tgz#dbc19154f558690150a23953a0adbd035be45fc7" From 59da2df7b201f7b3f05d9ecb6e01c7311563caed Mon Sep 17 00:00:00 2001 From: Bryce Buchanan <75274611+bryce-b@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:22:31 -0800 Subject: [PATCH 18/51] [Cypress] Transaction details tests fix (#199391) ## Summary This PR re-enables the Transaction details cypress test, which was disabled due to flakiness. The majority of the tests in this file needed an increased timeout to run correctly. However, the 'Show top errors table' test remains disabled, because it will consistently fail in buildkite executing `.contains('a', '[MockError] Foo')`. This is unreproducible locally, and no amount of refactoring or re-configuring seems to affect the outcome. ### Checklist Delete any items that are not applicable to this PR. - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed --- .../e2e/transaction_details/transaction_details.cy.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/transaction_details/transaction_details.cy.ts b/x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/transaction_details/transaction_details.cy.ts index af23fc8a2ad7e..0ef8fb40ceb38 100644 --- a/x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/transaction_details/transaction_details.cy.ts +++ b/x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/transaction_details/transaction_details.cy.ts @@ -15,8 +15,7 @@ const timeRange = { rangeFrom: start, rangeTo: end, }; -// flaky -describe.skip('Transaction details', () => { +describe('Transaction details', () => { before(() => { synthtrace.index( opbeans({ @@ -34,7 +33,7 @@ describe.skip('Transaction details', () => { cy.loginAsViewerUser(); }); - it('shows transaction name and transaction charts', () => { + it('shows transaction name and transaction charts', { defaultCommandTimeout: 60000 }, () => { cy.intercept('GET', '/internal/apm/services/opbeans-java/transactions/charts/latency?*').as( 'transactionLatencyRequest' ); @@ -60,7 +59,7 @@ describe.skip('Transaction details', () => { '@transactionThroughputRequest', '@transactionFailureRateRequest', ], - { timeout: 30000 } + { timeout: 60000 } ).spread((latencyInterception, throughputInterception, failureRateInterception) => { expect(latencyInterception.request.query.transactionName).to.be.eql('GET /api/product'); @@ -106,7 +105,9 @@ describe.skip('Transaction details', () => { ); cy.contains('Create SLO'); }); - it('shows top errors table', () => { + + // flaky + it.skip('shows top errors table', () => { cy.visitKibana( `/app/apm/services/opbeans-java/transactions/view?${new URLSearchParams({ ...timeRange, From 55a0df9c5ef9a4627838fd3a6e7b1e516197a672 Mon Sep 17 00:00:00 2001 From: Sergi Romeu Date: Tue, 3 Dec 2024 18:34:22 +0100 Subject: [PATCH 19/51] [Infra] Add Flyout to table view in Infrastructure Inventory (#202646) ## Summary Closes #201138 This PR adds a Flyout when clicking an item in the table view for the Infrastructure Inventory, as we had it in the Map view but the table view was missing. ### How to test 1. Start Kibana and fill it with data, `node scripts/synthtrace infra_hosts_with_apm_hosts` can be used if you don't have data. 2. Navigate to Infrastructure Inventory 3. Change the view to table view 4. Click on an item, the flyout should appear ### Screenshots Before|After -|- ![image](https://github.com/user-attachments/assets/585a4036-364e-447c-9f27-130e594b7a69)|![image](https://github.com/user-attachments/assets/4fc42ee7-27ed-4035-b24d-6bcc305407e5) --- .../components/nodes_overview.tsx | 12 ++++++++++++ .../inventory_view/components/table_view.tsx | 17 +++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/nodes_overview.tsx b/x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/nodes_overview.tsx index d42670f190fde..c2493df3a3968 100644 --- a/x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/nodes_overview.tsx +++ b/x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/nodes_overview.tsx @@ -145,6 +145,18 @@ export const NodesOverview = ({ currentTime={currentTime} onFilter={handleDrilldown} /> + {nodeType === assetType && detailsItemId && ( + + )} ); } diff --git a/x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/table_view.tsx b/x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/table_view.tsx index 182c74c124fd9..e41bf377e40e1 100644 --- a/x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/table_view.tsx +++ b/x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/table_view.tsx @@ -17,6 +17,7 @@ import { InfraWaffleMapNode, InfraWaffleMapOptions } from '../../../../common/in import { fieldToName } from '../lib/field_to_display_name'; import { NodeContextMenu } from './waffle/node_context_menu'; import { SnapshotNode, SnapshotNodePath } from '../../../../../common/http_api/snapshot_api'; +import { useAssetDetailsFlyoutState } from '../hooks/use_asset_details_flyout_url_state'; interface Props { nodes: SnapshotNode[]; @@ -49,6 +50,16 @@ export const TableView = (props: Props) => { const { nodes, options, formatter, currentTime, nodeType } = props; const [openPopoverId, setOpenPopoverId] = useState(null); + const [_, setFlyoutUrlState] = useAssetDetailsFlyoutState(); + const isFlyoutMode = nodeType === 'host' || nodeType === 'container'; + + const toggleAssetPopover = (uniqueID: string, nodeId: string) => { + if (isFlyoutMode) { + setFlyoutUrlState({ detailsItemId: nodeId, assetType: nodeType }); + } else { + setOpenPopoverId(uniqueID); + } + }; const closePopover = () => setOpenPopoverId(null); @@ -69,14 +80,14 @@ export const TableView = (props: Props) => { setOpenPopoverId(uniqueID)} + onClick={() => toggleAssetPopover(uniqueID, item.node.id)} > {value} ); - return ( + return !isFlyoutMode ? ( { options={options} /> + ) : ( + button ); }, }, From 6e5fc696a6f940d21af4ec5e4fd46d73a1007b71 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 3 Dec 2024 12:49:49 -0500 Subject: [PATCH 20/51] [Fleet] Use index.mapping.source.mode instead of _source.mode (#202729) --- .../elasticsearch/template/install.test.ts | 28 ++++++++++--------- .../epm/elasticsearch/template/install.ts | 18 ++++++------ .../epm/elasticsearch/template/template.ts | 6 ++-- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index 3b4b74cf772e5..f735e8638b583 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -124,12 +124,13 @@ describe('EPM index template install', () => { const packageTemplate = componentTemplates['metrics-package.dataset@package'].template; - if (!('mappings' in packageTemplate)) { + if (!('settings' in packageTemplate)) { throw new Error('no mappings on package template'); } - expect(packageTemplate.mappings).toHaveProperty('_source'); - expect(packageTemplate.mappings._source).toEqual({ mode: 'synthetic' }); + expect(packageTemplate.settings?.index?.mapping).toHaveProperty('source'); + // @ts-expect-error esclient mapping out-of-date + expect(packageTemplate.settings?.index?.mapping?.source).toEqual({ mode: 'synthetic' }); }); it('tests prepareTemplate to set source mode to synthetics if index_mode:time_series', async () => { @@ -154,12 +155,13 @@ describe('EPM index template install', () => { const packageTemplate = componentTemplates['metrics-package.dataset@package'].template; - if (!('mappings' in packageTemplate)) { - throw new Error('no mappings on package template'); + if (!('settings' in packageTemplate)) { + throw new Error('no settings on package template'); } - expect(packageTemplate.mappings).toHaveProperty('_source'); - expect(packageTemplate.mappings._source).toEqual({ mode: 'synthetic' }); + expect(packageTemplate.settings?.index?.mapping).toHaveProperty('source'); + // @ts-expect-error esclient mapping out-of-date + expect(packageTemplate.settings?.index?.mapping?.source).toEqual({ mode: 'synthetic' }); }); it('tests prepareTemplate to not set source mode to synthetics if index_mode:time_series and user disabled synthetic', async () => { @@ -193,11 +195,11 @@ describe('EPM index template install', () => { const packageTemplate = componentTemplates['metrics-package.dataset@package'].template; - if (!('mappings' in packageTemplate)) { - throw new Error('no mappings on package template'); + if (!('settings' in packageTemplate)) { + throw new Error('no settings on package template'); } - expect(packageTemplate.mappings).not.toHaveProperty('_source'); + expect(packageTemplate.settings?.index?.mapping).not.toHaveProperty('source'); }); it('tests prepareTemplate to not set source mode to synthetics if specified but user disabled it', async () => { @@ -231,11 +233,11 @@ describe('EPM index template install', () => { const packageTemplate = componentTemplates['metrics-package.dataset@package'].template; - if (!('mappings' in packageTemplate)) { - throw new Error('no mappings on package template'); + if (!('settings' in packageTemplate)) { + throw new Error('no settings on package template'); } - expect(packageTemplate.mappings).not.toHaveProperty('_source'); + expect(packageTemplate.settings?.index?.mapping).not.toHaveProperty('source'); }); it('tests prepareTemplate to set index_mode time series if index_mode:time_series', async () => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index a15e8cb71923c..f17a05a6837aa 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -409,6 +409,14 @@ export function buildComponentTemplates(params: { templateSettings.index?.mapping?.total_fields?.limit ), }, + ...(templateSettings.index?.mapping?.source || sourceModeSynthetic + ? { + source: { + ...templateSettings.index?.mapping?.source, + ...(sourceModeSynthetic ? { mode: 'synthetic' } : {}), + }, + } + : {}), }, }, }, @@ -418,15 +426,7 @@ export function buildComponentTemplates(params: { ? { runtime: mappingsRuntimeFields } : {}), dynamic_templates: mappingsDynamicTemplates.length ? mappingsDynamicTemplates : undefined, - ...omit(indexTemplateMappings, 'properties', 'dynamic_templates', '_source', 'runtime'), - ...(indexTemplateMappings?._source || sourceModeSynthetic - ? { - _source: { - ...indexTemplateMappings?._source, - ...(sourceModeSynthetic ? { mode: 'synthetic' } : {}), - }, - } - : {}), + ...omit(indexTemplateMappings, 'properties', 'dynamic_templates', 'runtime'), }, ...(lifecycle ? { lifecycle } : {}), }, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 57ce30550af68..d44cd57a2c5ba 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -1031,8 +1031,8 @@ const updateExistingDataStream = async ({ const existingDsConfig = Object.values(existingDs); const currentBackingIndexConfig = existingDsConfig.at(-1); const currentIndexMode = currentBackingIndexConfig?.settings?.index?.mode; - // @ts-expect-error Property 'mode' does not exist on type 'MappingSourceField' - const currentSourceType = currentBackingIndexConfig.mappings?._source?.mode; + // @ts-expect-error Property 'source.mode' does not exist on type 'IndicesMappingLimitSettings' + const currentSourceType = currentBackingIndexConfig?.settings?.index?.mapping?.source?.mode; let settings: IndicesIndexSettings; let mappings: MappingTypeMapping = {}; @@ -1141,7 +1141,7 @@ const updateExistingDataStream = async ({ // Trigger a rollover if the index mode or source type has changed if ( currentIndexMode !== settings?.index?.mode || - currentSourceType !== mappings?._source?.mode || + currentSourceType !== settings?.index?.source?.mode || dynamicDimensionMappingsChanged ) { if (options?.skipDataStreamRollover === true) { From a662233d8b967d022a32fc9379d338684bf6a413 Mon Sep 17 00:00:00 2001 From: Ievgen Sorokopud Date: Tue, 3 Dec 2024 18:50:10 +0100 Subject: [PATCH 21/51] [Rules migration] Add pagination functionality to rules migration table (#11313) (#202494) ## Summary [Internal link](https://github.com/elastic/security-team/issues/10820) to the feature details With these changes we add pagination functionality for the rules migration table. This way we will improve the performance within the page. Also, added as part of these PR: * moved `install` and `install_translated` routes to the `rules/api` folder; before those were located in `rules/api/rules` and made confusion * a new `translation_stats` route to return stats for the specific migration about the translated rules, like `total` number of the rules, and number of `prebuilt`, `custom` and `installable` rules * add `Updated` table column * small UI fixes: * use correct icon for "SIEM rule migration" * do not remove "Install translated rules" button and rather disable it when there are no installable rules * do not allow user to update translation status via UI --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../common/api/quickstart_client.gen.ts | 27 +++++ .../common/siem_migrations/constants.ts | 2 + .../model/api/rules/rule_migration.gen.ts | 31 ++++- .../api/rules/rule_migration.schema.yaml | 58 +++++++++- .../model/rule_migration.gen.ts | 32 ++++++ .../model/rule_migration.schema.yaml | 34 +++++- .../public/common/icons/siem_migrations.tsx | 39 +++++++ .../public/siem_migrations/links.ts | 4 +- .../public/siem_migrations/rules/api/api.ts | 44 ++++++- .../hooks/use_get_migration_rules_query.ts | 38 ++++-- ...e_get_migration_translation_stats_query.ts | 60 ++++++++++ .../use_install_migration_rules_mutation.ts | 4 + ...ll_translated_migration_rules_mutation.ts} | 6 +- .../components/rules_table/bulk_actions.tsx | 34 +++--- .../rules/components/rules_table/filters.tsx | 54 --------- .../rules/components/rules_table/index.tsx | 108 +++++++++--------- .../components/rules_table/search_field.tsx | 57 +++++++++ .../components/rules_table/translations.ts | 37 ++++-- .../components/rules_table_columns/index.tsx | 1 + .../rules_table_columns/translations.ts | 7 ++ .../rules_table_columns/updated.tsx | 26 +++++ .../rules/components/status_badge/index.tsx | 2 +- .../translation_tab/index.tsx | 9 +- .../hooks/use_filter_rules_to_install.ts | 33 ------ .../rules/hooks/use_rules_table_columns.tsx | 2 + .../rules/logic/translations.ts | 7 ++ .../rules/logic/use_get_migration_rules.ts | 9 +- .../use_get_migration_translation_stats.ts | 20 ++++ ...use_install_translated_migration_rules.ts} | 6 +- .../siem_migrations/rules/utils/helpers.tsx | 18 ++- .../rules/utils/translations.ts | 10 ++ .../lib/siem_migrations/rules/api/get.ts | 26 ++++- .../lib/siem_migrations/rules/api/index.ts | 6 +- .../rules/api/{rules => }/install.ts | 12 +- .../api/{rules => }/install_translated.ts | 12 +- .../rules/api/translation_stats.ts | 56 +++++++++ .../rules/api/util/installation.ts | 2 +- .../data/rule_migrations_data_rules_client.ts | 99 +++++++++++++--- .../rules/data/rule_migrations_field_maps.ts | 8 +- .../services/security_solution_api.gen.ts | 34 +++++- 40 files changed, 839 insertions(+), 235 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/common/icons/siem_migrations.tsx create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_get_migration_translation_stats_query.ts rename x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/{use_install_all_migration_rules_mutation.ts => use_install_translated_migration_rules_mutation.ts} (81%) delete mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/filters.tsx create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/search_field.tsx create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/updated.tsx delete mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/hooks/use_filter_rules_to_install.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_translation_stats.ts rename x-pack/plugins/security_solution/public/siem_migrations/rules/logic/{use_install_all_migration_rules.ts => use_install_translated_migration_rules.ts} (65%) rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/{rules => }/install.ts (85%) rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/{rules => }/install_translated.ts (84%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/translation_stats.ts diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts index d86a04f343a89..b03a81b3b2249 100644 --- a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts +++ b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts @@ -352,6 +352,7 @@ import type { CreateRuleMigrationRequestBodyInput, CreateRuleMigrationResponse, GetAllStatsRuleMigrationResponse, + GetRuleMigrationRequestQueryInput, GetRuleMigrationRequestParamsInput, GetRuleMigrationResponse, GetRuleMigrationResourcesRequestQueryInput, @@ -359,6 +360,8 @@ import type { GetRuleMigrationResourcesResponse, GetRuleMigrationStatsRequestParamsInput, GetRuleMigrationStatsResponse, + GetRuleMigrationTranslationStatsRequestParamsInput, + GetRuleMigrationTranslationStatsResponse, InstallMigrationRulesRequestParamsInput, InstallMigrationRulesRequestBodyInput, InstallMigrationRulesResponse, @@ -1415,6 +1418,8 @@ finalize it. [ELASTIC_HTTP_VERSION_HEADER]: '1', }, method: 'GET', + + query: props.query, }) .catch(catchAxiosErrorFormatAndThrow); } @@ -1453,6 +1458,24 @@ finalize it. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Retrieves the translation stats of a SIEM rules migration using the migration id provided + */ + async getRuleMigrationTranslationStats(props: GetRuleMigrationTranslationStatsProps) { + this.log.info(`${new Date().toISOString()} Calling API GetRuleMigrationTranslationStats`); + return this.kbnClient + .request({ + path: replaceParams( + '/internal/siem_migrations/rules/{migration_id}/translation_stats', + props.params + ), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'GET', + }) + .catch(catchAxiosErrorFormatAndThrow); + } /** * Get the details of an existing saved Timeline or Timeline template. */ @@ -2334,6 +2357,7 @@ export interface GetRuleExecutionResultsProps { params: GetRuleExecutionResultsRequestParamsInput; } export interface GetRuleMigrationProps { + query: GetRuleMigrationRequestQueryInput; params: GetRuleMigrationRequestParamsInput; } export interface GetRuleMigrationResourcesProps { @@ -2343,6 +2367,9 @@ export interface GetRuleMigrationResourcesProps { export interface GetRuleMigrationStatsProps { params: GetRuleMigrationStatsRequestParamsInput; } +export interface GetRuleMigrationTranslationStatsProps { + params: GetRuleMigrationTranslationStatsRequestParamsInput; +} export interface GetTimelineProps { query: GetTimelineRequestQueryInput; } diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts index 565091e39a8db..a910cb90f1664 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts @@ -15,6 +15,8 @@ export const SIEM_RULE_MIGRATION_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration export const SIEM_RULE_MIGRATION_START_PATH = `${SIEM_RULE_MIGRATION_PATH}/start` as const; export const SIEM_RULE_MIGRATION_RETRY_PATH = `${SIEM_RULE_MIGRATION_PATH}/retry` as const; export const SIEM_RULE_MIGRATION_STATS_PATH = `${SIEM_RULE_MIGRATION_PATH}/stats` as const; +export const SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH = + `${SIEM_RULE_MIGRATION_PATH}/translation_stats` as const; export const SIEM_RULE_MIGRATION_STOP_PATH = `${SIEM_RULE_MIGRATION_PATH}/stop` as const; export const SIEM_RULE_MIGRATION_INSTALL_PATH = `${SIEM_RULE_MIGRATION_PATH}/install` as const; export const SIEM_RULE_MIGRATION_INSTALL_TRANSLATED_PATH = diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts index 77a0fc94408f9..aa69f3b3c27f0 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts @@ -24,6 +24,7 @@ import { RuleMigrationComments, RuleMigrationTaskStats, RuleMigration, + RuleMigrationTranslationStats, RuleMigrationResourceData, RuleMigrationResourceType, RuleMigrationResource, @@ -44,6 +45,13 @@ export const CreateRuleMigrationResponse = z.object({ export type GetAllStatsRuleMigrationResponse = z.infer; export const GetAllStatsRuleMigrationResponse = z.array(RuleMigrationTaskStats); +export type GetRuleMigrationRequestQuery = z.infer; +export const GetRuleMigrationRequestQuery = z.object({ + page: z.coerce.number().optional(), + per_page: z.coerce.number().optional(), + search_term: z.string().optional(), +}); +export type GetRuleMigrationRequestQueryInput = z.input; export type GetRuleMigrationRequestParams = z.infer; export const GetRuleMigrationRequestParams = z.object({ @@ -52,7 +60,13 @@ export const GetRuleMigrationRequestParams = z.object({ export type GetRuleMigrationRequestParamsInput = z.input; export type GetRuleMigrationResponse = z.infer; -export const GetRuleMigrationResponse = z.array(RuleMigration); +export const GetRuleMigrationResponse = z.object({ + /** + * The total number of rules in migration. + */ + total: z.number(), + data: z.array(RuleMigration), +}); export type GetRuleMigrationResourcesRequestQuery = z.infer< typeof GetRuleMigrationResourcesRequestQuery >; @@ -88,6 +102,21 @@ export type GetRuleMigrationStatsRequestParamsInput = z.input< export type GetRuleMigrationStatsResponse = z.infer; export const GetRuleMigrationStatsResponse = RuleMigrationTaskStats; +export type GetRuleMigrationTranslationStatsRequestParams = z.infer< + typeof GetRuleMigrationTranslationStatsRequestParams +>; +export const GetRuleMigrationTranslationStatsRequestParams = z.object({ + migration_id: NonEmptyString, +}); +export type GetRuleMigrationTranslationStatsRequestParamsInput = z.input< + typeof GetRuleMigrationTranslationStatsRequestParams +>; + +export type GetRuleMigrationTranslationStatsResponse = z.infer< + typeof GetRuleMigrationTranslationStatsResponse +>; +export const GetRuleMigrationTranslationStatsResponse = RuleMigrationTranslationStats; + export type InstallMigrationRulesRequestParams = z.infer; export const InstallMigrationRulesRequestParams = z.object({ migration_id: NonEmptyString, diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml index f57a809bb204e..a062b75d41699 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml @@ -185,15 +185,40 @@ paths: schema: description: The migration id to start $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' + - name: page + in: query + required: false + schema: + type: number + - name: per_page + in: query + required: false + schema: + type: number + - name: search_term + in: query + required: false + schema: + type: string + responses: 200: description: Indicates rule migration have been retrieved correctly. content: application/json: schema: - type: array - items: - $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigration' + type: object + required: + - total + - data + properties: + total: + type: number + description: The total number of rules in migration. + data: + type: array + items: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigration' 204: description: Indicates the migration id was not found. @@ -256,7 +281,7 @@ paths: in: path required: true schema: - description: The migration id to start + description: The migration id to fetch stats for $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' responses: 200: @@ -268,6 +293,31 @@ paths: 204: description: Indicates the migration id was not found. + /internal/siem_migrations/rules/{migration_id}/translation_stats: + get: + summary: Gets a rule migration translation stats + operationId: GetRuleMigrationTranslationStats + x-codegen-enabled: true + description: Retrieves the translation stats of a SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + description: The migration id to fetch translation stats for + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' + responses: + 200: + description: Indicates the migration stats has been retrieved correctly. + content: + application/json: + schema: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationTranslationStats' + 204: + description: Indicates the migration id was not found. + /internal/siem_migrations/rules/{migration_id}/stop: put: summary: Stops an existing rule migration diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 82e3c5549fd86..61706077d9549 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -242,6 +242,38 @@ export const RuleMigrationTaskStats = z.object({ last_updated_at: z.string(), }); +/** + * The rule migration translation stats object. + */ +export type RuleMigrationTranslationStats = z.infer; +export const RuleMigrationTranslationStats = z.object({ + /** + * The migration id + */ + id: NonEmptyString, + /** + * The rules migration translation stats. + */ + rules: z.object({ + /** + * The total number of rules to migrate. + */ + total: z.number().int(), + /** + * The number of rules that matched Elastic prebuilt rules. + */ + prebuilt: z.number().int(), + /** + * The number of rules that did not match Elastic prebuilt rules and will be installed as custom rules. + */ + custom: z.number().int(), + /** + * The number of rules that can be installed. + */ + installable: z.number().int(), + }), +}); + /** * The type of the rule migration resource. */ diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index 82892b4fa0722..fdcbb7b04515a 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -197,7 +197,39 @@ components: - running - stopped - finished - + + RuleMigrationTranslationStats: + type: object + description: The rule migration translation stats object. + required: + - id + - rules + properties: + id: + description: The migration id + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' + rules: + type: object + description: The rules migration translation stats. + required: + - total + - prebuilt + - custom + - installable + properties: + total: + type: integer + description: The total number of rules to migrate. + prebuilt: + type: integer + description: The number of rules that matched Elastic prebuilt rules. + custom: + type: integer + description: The number of rules that did not match Elastic prebuilt rules and will be installed as custom rules. + installable: + type: integer + description: The number of rules that can be installed. + RuleMigrationTranslationResult: type: string description: The rule translation result. diff --git a/x-pack/plugins/security_solution/public/common/icons/siem_migrations.tsx b/x-pack/plugins/security_solution/public/common/icons/siem_migrations.tsx new file mode 100644 index 0000000000000..072b0acf9d30a --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/icons/siem_migrations.tsx @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { SVGProps } from 'react'; +import React from 'react'; +export const SiemMigrationsIcon: React.FC> = ({ ...props }) => ( + + + + + + + + + + + + + + +); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/links.ts b/x-pack/plugins/security_solution/public/siem_migrations/links.ts index 34db8a357785a..a98f823957acf 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/links.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/links.ts @@ -13,7 +13,7 @@ import { } from '../../common/constants'; import { SIEM_MIGRATIONS_RULES } from '../app/translations'; import type { LinkItem } from '../common/links/types'; -import { IconConsoleCloud } from '../common/icons/console_cloud'; +import { SiemMigrationsIcon } from '../common/icons/siem_migrations'; export const siemMigrationsLinks: LinkItem = { id: SecurityPageName.siemMigrationsRules, @@ -21,7 +21,7 @@ export const siemMigrationsLinks: LinkItem = { description: i18n.translate('xpack.securitySolution.appLinks.siemMigrationsRulesDescription', { defaultMessage: 'SIEM Rules Migrations.', }), - landingIcon: IconConsoleCloud, + landingIcon: SiemMigrationsIcon, path: SIEM_MIGRATIONS_RULES_PATH, capabilities: [`${SERVER_APP_ID}.show`], skipUrlState: true, diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/api.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/api.ts index 3b7605e032259..160d78f1edbb6 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/api.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/api.ts @@ -15,10 +15,12 @@ import { SIEM_RULE_MIGRATION_INSTALL_PATH, SIEM_RULE_MIGRATION_PATH, SIEM_RULE_MIGRATION_START_PATH, + SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH, } from '../../../../common/siem_migrations/constants'; import type { GetAllStatsRuleMigrationResponse, GetRuleMigrationResponse, + GetRuleMigrationTranslationStatsResponse, InstallTranslatedMigrationRulesResponse, InstallMigrationRulesResponse, StartRuleMigrationRequestBody, @@ -67,6 +69,31 @@ export const startRuleMigration = async ({ ); }; +/** + * Retrieves the translation stats for the migraion. + * + * @param migrationId `id` of the migration to retrieve translation stats for + * @param signal AbortSignal for cancelling request + * + * @throws An error if response is not OK + */ +export const getRuleMigrationTranslationStats = async ({ + migrationId, + signal, +}: { + migrationId: string; + signal: AbortSignal | undefined; +}): Promise => { + return KibanaServices.get().http.fetch( + replaceParams(SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH, { migration_id: migrationId }), + { + method: 'GET', + version: '1', + signal, + } + ); +}; + /** * Retrieves all the migration rule documents of a specific migration. * @@ -77,14 +104,29 @@ export const startRuleMigration = async ({ */ export const getRuleMigrations = async ({ migrationId, + page, + perPage, + searchTerm, signal, }: { migrationId: string; + page?: number; + perPage?: number; + searchTerm?: string; signal: AbortSignal | undefined; }): Promise => { return KibanaServices.get().http.fetch( replaceParams(SIEM_RULE_MIGRATION_PATH, { migration_id: migrationId }), - { method: 'GET', version: '1', signal } + { + method: 'GET', + version: '1', + query: { + page, + per_page: perPage, + search_term: searchTerm, + }, + signal, + } ); }; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_get_migration_rules_query.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_get_migration_rules_query.ts index fece8f8c3ca07..138b2a31e9727 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_get_migration_rules_query.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_get_migration_rules_query.ts @@ -9,26 +9,46 @@ import type { UseQueryOptions } from '@tanstack/react-query'; import { useQuery, useQueryClient } from '@tanstack/react-query'; import { replaceParams } from '@kbn/openapi-common/shared'; import { useCallback } from 'react'; +import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { DEFAULT_QUERY_OPTIONS } from './constants'; import { getRuleMigrations } from '../api'; -import type { GetRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATION_PATH } from '../../../../../common/siem_migrations/constants'; +interface UseGetMigrationRulesQueryProps { + migrationId: string; + page?: number; + perPage?: number; + searchTerm?: string; +} + +export interface MigrationRulesQueryResponse { + ruleMigrations: RuleMigration[]; + total: number; +} + export const useGetMigrationRulesQuery = ( - migrationId: string, - options?: UseQueryOptions + queryArgs: UseGetMigrationRulesQueryProps, + queryOptions?: UseQueryOptions< + MigrationRulesQueryResponse, + Error, + MigrationRulesQueryResponse, + [...string[], UseGetMigrationRulesQueryProps] + > ) => { + const { migrationId } = queryArgs; const SPECIFIC_MIGRATION_PATH = replaceParams(SIEM_RULE_MIGRATION_PATH, { migration_id: migrationId, }); - return useQuery( - ['GET', SPECIFIC_MIGRATION_PATH], + return useQuery( + ['GET', SPECIFIC_MIGRATION_PATH, queryArgs], async ({ signal }) => { - return getRuleMigrations({ migrationId, signal }); + const response = await getRuleMigrations({ signal, ...queryArgs }); + + return { ruleMigrations: response.data, total: response.total }; }, { ...DEFAULT_QUERY_OPTIONS, - ...options, + ...queryOptions, } ); }; @@ -47,6 +67,10 @@ export const useInvalidateGetMigrationRulesQuery = (migrationId: string) => { }); return useCallback(() => { + /** + * Invalidate all queries that start with SPECIFIC_MIGRATION_PATH. This + * includes the in-memory query cache and paged query cache. + */ queryClient.invalidateQueries(['GET', SPECIFIC_MIGRATION_PATH], { refetchType: 'active', }); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_get_migration_translation_stats_query.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_get_migration_translation_stats_query.ts new file mode 100644 index 0000000000000..e58b9be20d19c --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_get_migration_translation_stats_query.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseQueryOptions } from '@tanstack/react-query'; +import { useQuery, useQueryClient } from '@tanstack/react-query'; +import { replaceParams } from '@kbn/openapi-common/shared'; +import { useCallback } from 'react'; +import { DEFAULT_QUERY_OPTIONS } from './constants'; +import { getRuleMigrationTranslationStats } from '../api'; +import type { GetRuleMigrationTranslationStatsResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH } from '../../../../../common/siem_migrations/constants'; + +export const useGetMigrationTranslationStatsQuery = ( + migrationId: string, + options?: UseQueryOptions +) => { + const SPECIFIC_MIGRATION_TRANSLATION_PATH = replaceParams( + SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH, + { + migration_id: migrationId, + } + ); + return useQuery( + ['GET', SPECIFIC_MIGRATION_TRANSLATION_PATH], + async ({ signal }) => { + return getRuleMigrationTranslationStats({ migrationId, signal }); + }, + { + ...DEFAULT_QUERY_OPTIONS, + ...options, + } + ); +}; + +/** + * We should use this hook to invalidate the translation stats cache. For + * example, rule migrations mutations, like installing a rule, should lead to cache invalidation. + * + * @returns A translation stats cache invalidation callback + */ +export const useInvalidateGetMigrationTranslationStatsQuery = (migrationId: string) => { + const queryClient = useQueryClient(); + + const SPECIFIC_MIGRATION_TRANSLATION_PATH = replaceParams( + SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH, + { + migration_id: migrationId, + } + ); + + return useCallback(() => { + queryClient.invalidateQueries(['GET', SPECIFIC_MIGRATION_TRANSLATION_PATH], { + refetchType: 'active', + }); + }, [SPECIFIC_MIGRATION_TRANSLATION_PATH, queryClient]); +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_migration_rules_mutation.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_migration_rules_mutation.ts index 6aaff55e24513..536f9e772e5d8 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_migration_rules_mutation.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_migration_rules_mutation.ts @@ -10,6 +10,7 @@ import type { InstallMigrationRulesResponse } from '../../../../../common/siem_m import { SIEM_RULE_MIGRATION_INSTALL_PATH } from '../../../../../common/siem_migrations/constants'; import { installMigrationRules } from '../api'; import { useInvalidateGetMigrationRulesQuery } from './use_get_migration_rules_query'; +import { useInvalidateGetMigrationTranslationStatsQuery } from './use_get_migration_translation_stats_query'; export const INSTALL_MIGRATION_RULES_MUTATION_KEY = ['POST', SIEM_RULE_MIGRATION_INSTALL_PATH]; @@ -18,6 +19,8 @@ export const useInstallMigrationRulesMutation = ( options?: UseMutationOptions ) => { const invalidateGetRuleMigrationsQuery = useInvalidateGetMigrationRulesQuery(migrationId); + const invalidateGetMigrationTranslationStatsQuery = + useInvalidateGetMigrationTranslationStatsQuery(migrationId); return useMutation( (ids: string[]) => installMigrationRules({ migrationId, ids }), @@ -26,6 +29,7 @@ export const useInstallMigrationRulesMutation = ( mutationKey: INSTALL_MIGRATION_RULES_MUTATION_KEY, onSettled: (...args) => { invalidateGetRuleMigrationsQuery(); + invalidateGetMigrationTranslationStatsQuery(); if (options?.onSettled) { options.onSettled(...args); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_all_migration_rules_mutation.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_translated_migration_rules_mutation.ts similarity index 81% rename from x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_all_migration_rules_mutation.ts rename to x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_translated_migration_rules_mutation.ts index f946dc165450f..3c1dda0b457c4 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_all_migration_rules_mutation.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/hooks/use_install_translated_migration_rules_mutation.ts @@ -10,17 +10,20 @@ import type { InstallTranslatedMigrationRulesResponse } from '../../../../../com import { SIEM_RULE_MIGRATION_INSTALL_TRANSLATED_PATH } from '../../../../../common/siem_migrations/constants'; import { installTranslatedMigrationRules } from '../api'; import { useInvalidateGetMigrationRulesQuery } from './use_get_migration_rules_query'; +import { useInvalidateGetMigrationTranslationStatsQuery } from './use_get_migration_translation_stats_query'; export const INSTALL_ALL_MIGRATION_RULES_MUTATION_KEY = [ 'POST', SIEM_RULE_MIGRATION_INSTALL_TRANSLATED_PATH, ]; -export const useInstallAllMigrationRulesMutation = ( +export const useInstallTranslatedMigrationRulesMutation = ( migrationId: string, options?: UseMutationOptions ) => { const invalidateGetRuleMigrationsQuery = useInvalidateGetMigrationRulesQuery(migrationId); + const invalidateGetMigrationTranslationStatsQuery = + useInvalidateGetMigrationTranslationStatsQuery(migrationId); return useMutation( () => installTranslatedMigrationRules({ migrationId }), @@ -29,6 +32,7 @@ export const useInstallAllMigrationRulesMutation = ( mutationKey: INSTALL_ALL_MIGRATION_RULES_MUTATION_KEY, onSettled: (...args) => { invalidateGetRuleMigrationsQuery(); + invalidateGetMigrationTranslationStatsQuery(); if (options?.onSettled) { options.onSettled(...args); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/bulk_actions.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/bulk_actions.tsx index df6d01d876fce..a58681b6e771f 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/bulk_actions.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/bulk_actions.tsx @@ -28,9 +28,9 @@ export const BulkActions: React.FC = React.memo( installTranslatedRule, installSelectedRule, }) => { - const showInstallTranslatedRulesButton = numberOfTranslatedRules > 0; + const disableInstallTranslatedRulesButton = isTableLoading || !numberOfTranslatedRules; const showInstallSelectedRulesButton = - showInstallTranslatedRulesButton && numberOfSelectedRules > 0; + disableInstallTranslatedRulesButton && numberOfSelectedRules > 0; return ( {showInstallSelectedRulesButton ? ( @@ -46,21 +46,21 @@ export const BulkActions: React.FC = React.memo( ) : null} - {showInstallTranslatedRulesButton ? ( - - - {i18n.INSTALL_ALL_RULES(numberOfTranslatedRules)} - {isTableLoading && } - - - ) : null} + + + {numberOfTranslatedRules > 0 + ? i18n.INSTALL_TRANSLATED_RULES(numberOfTranslatedRules) + : i18n.INSTALL_TRANSLATED_RULES_EMPTY_STATE} + {isTableLoading && } + + ); } diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/filters.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/filters.tsx deleted file mode 100644 index 25dffc64cccc5..0000000000000 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/filters.tsx +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiFlexGroup } from '@elastic/eui'; -import type { Dispatch, SetStateAction } from 'react'; -import React, { useCallback } from 'react'; -import * as i18n from './translations'; -import { RuleSearchField } from '../../../../detection_engine/rule_management_ui/components/rules_table/rules_table_filters/rule_search_field'; -import type { TableFilterOptions } from '../../hooks/use_filter_rules_to_install'; - -export interface FiltersComponentProps { - /** - * Currently selected table filter - */ - filterOptions: TableFilterOptions; - - /** - * Handles filter options changes - */ - setFilterOptions: Dispatch>; -} - -/** - * Collection of filters for filtering data within the SIEM Rules Migrations table. - * Contains search bar and tag selection - */ -const FiltersComponent: React.FC = ({ filterOptions, setFilterOptions }) => { - const handleOnSearch = useCallback( - (filterString: string) => { - setFilterOptions((filters) => ({ - ...filters, - filter: filterString.trim(), - })); - }, - [setFilterOptions] - ); - - return ( - - - - ); -}; - -export const Filters = React.memo(FiltersComponent); -Filters.displayName = 'Filters'; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx index 16f93a1cdebaf..82793d3e1fd8c 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx @@ -5,33 +5,30 @@ * 2.0. */ +import type { CriteriaWithPagination } from '@elastic/eui'; import { - EuiInMemoryTable, EuiSkeletonLoading, - EuiProgress, EuiSkeletonTitle, EuiSkeletonText, EuiFlexGroup, EuiFlexItem, EuiSpacer, + EuiBasicTable, } from '@elastic/eui'; import React, { useCallback, useMemo, useState } from 'react'; import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -import { - RULES_TABLE_INITIAL_PAGE_SIZE, - RULES_TABLE_PAGE_SIZE_OPTIONS, -} from '../../../../detection_engine/rule_management_ui/components/rules_table/constants'; import { NoItemsMessage } from './no_items_message'; -import { Filters } from './filters'; import { useRulesTableColumns } from '../../hooks/use_rules_table_columns'; -import type { TableFilterOptions } from '../../hooks/use_filter_rules_to_install'; -import { useFilterRulesToInstall } from '../../hooks/use_filter_rules_to_install'; import { useRulePreviewFlyout } from '../../hooks/use_rule_preview_flyout'; import { useInstallMigrationRules } from '../../logic/use_install_migration_rules'; import { useGetMigrationRules } from '../../logic/use_get_migration_rules'; -import { useInstallAllMigrationRules } from '../../logic/use_install_all_migration_rules'; +import { useInstallTranslatedMigrationRules } from '../../logic/use_install_translated_migration_rules'; import { BulkActions } from './bulk_actions'; +import { useGetMigrationTranslationStats } from '../../logic/use_get_migration_translation_stats'; +import { SearchField } from './search_field'; + +const DEFAULT_PAGE_SIZE = 10; export interface RulesTableComponentProps { /** @@ -44,29 +41,47 @@ export interface RulesTableComponentProps { * Table Component for displaying SIEM rules migrations */ const RulesTableComponent: React.FC = ({ migrationId }) => { - const { data: ruleMigrations, isLoading: isDataLoading } = useGetMigrationRules(migrationId); + const [pageIndex, setPageIndex] = useState(0); + const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); + const [searchTerm, setSearchTerm] = useState(); + + const { data: translationStats, isLoading: isStatsLoading } = + useGetMigrationTranslationStats(migrationId); + + const { + data: { ruleMigrations, total } = { ruleMigrations: [], total: 0 }, + isLoading: isDataLoading, + } = useGetMigrationRules({ + migrationId, + page: pageIndex, + perPage: pageSize, + searchTerm, + }); const [selectedRuleMigrations, setSelectedRuleMigrations] = useState([]); - const [filterOptions, setFilterOptions] = useState({ - filter: '', - }); - - const filteredRuleMigrations = useFilterRulesToInstall({ - filterOptions, - ruleMigrations: ruleMigrations ?? [], - }); + const pagination = useMemo(() => { + return { + pageIndex, + pageSize, + totalItemCount: total, + }; + }, [pageIndex, pageSize, total]); + + const onTableChange = useCallback(({ page, sort }: CriteriaWithPagination) => { + if (page) { + setPageIndex(page.index); + setPageSize(page.size); + } + }, []); + + const handleOnSearch = useCallback((value: string) => { + setSearchTerm(value.trim()); + }, []); const { mutateAsync: installMigrationRules } = useInstallMigrationRules(migrationId); - const { mutateAsync: installAllMigrationRules } = useInstallAllMigrationRules(migrationId); - - const numberOfTranslatedRules = useMemo(() => { - return filteredRuleMigrations.filter( - (rule) => - !rule.elastic_rule?.id && - (rule.elastic_rule?.prebuilt_rule_id || rule.translation_result === 'full') - ).length; - }, [filteredRuleMigrations]); + const { mutateAsync: installTranslatedMigrationRules } = + useInstallTranslatedMigrationRules(migrationId); const [isTableLoading, setTableLoading] = useState(false); const installSingleRule = useCallback( @@ -85,12 +100,12 @@ const RulesTableComponent: React.FC = ({ migrationId } async (enable?: boolean) => { setTableLoading(true); try { - await installAllMigrationRules(); + await installTranslatedMigrationRules(); } finally { setTableLoading(false); } }, - [installAllMigrationRules] + [installTranslatedMigrationRules] ); const ruleActionsFactory = useCallback( @@ -105,8 +120,6 @@ const RulesTableComponent: React.FC = ({ migrationId } ruleActionsFactory, }); - const shouldShowProgress = isDataLoading; - const rulesColumns = useRulesTableColumns({ disableActions: isTableLoading, openMigrationRulePreview: openRulePreview, @@ -115,14 +128,6 @@ const RulesTableComponent: React.FC = ({ migrationId } return ( <> - {shouldShowProgress && ( - - )} = ({ migrationId } } loadedContent={ - !filteredRuleMigrations.length ? ( + !translationStats?.rules.total ? ( ) : ( <> - + - loading={isTableLoading} - items={filteredRuleMigrations} - sorting - pagination={{ - initialPageSize: RULES_TABLE_INITIAL_PAGE_SIZE, - pageSizeOptions: RULES_TABLE_PAGE_SIZE_OPTIONS, - }} + items={ruleMigrations} + pagination={pagination} + onChange={onTableChange} selection={{ selectable: () => true, onSelectionChange: setSelectedRuleMigrations, initialSelected: selectedRuleMigrations, }} - itemId="rule_id" - data-test-subj="rules-translation-table" + itemId={'id'} + data-test-subj={'rules-translation-table'} columns={rulesColumns} /> diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/search_field.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/search_field.tsx new file mode 100644 index 0000000000000..5bd18851ba595 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/search_field.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ChangeEvent } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; +import styled from 'styled-components'; +import { EuiFieldSearch, EuiFlexItem } from '@elastic/eui'; +import * as i18n from './translations'; + +const SearchBarWrapper = styled(EuiFlexItem)` + min-width: 200px; + & .euiPopover { + // This is needed to "cancel" styles passed down from EuiTourStep that + // interfere with EuiFieldSearch and don't allow it to take the full width + display: block; + } +`; + +interface SearchFieldProps { + initialValue?: string; + onSearch: (value: string) => void; + placeholder?: string; +} + +export const SearchField: React.FC = React.memo( + ({ initialValue, onSearch, placeholder }) => { + const [searchText, setSearchText] = useState(initialValue); + const handleChange = useCallback( + (e: ChangeEvent) => setSearchText(e.target.value), + [setSearchText] + ); + + useEffect(() => { + setSearchText(initialValue); + }, [initialValue]); + + return ( + + + + ); + } +); +SearchField.displayName = 'SearchField'; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/translations.ts index 3da9886659916..6803deb895d9b 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/translations.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table/translations.ts @@ -7,10 +7,17 @@ import { i18n } from '@kbn/i18n'; -export const SEARCH_PLACEHOLDER = i18n.translate( +export const SEARCH_MIGRATION_RULES = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.table.searchAriaLabel', + { + defaultMessage: 'Search migration rules', + } +); + +export const SEARCH_MIGRATION_RULES_PLACEHOLDER = i18n.translate( 'xpack.securitySolution.siemMigrations.rules.table.searchBarPlaceholder', { - defaultMessage: 'Search by rule name', + defaultMessage: 'Search by migration rule name', } ); @@ -42,12 +49,22 @@ export const INSTALL_SELECTED_RULES = (numberOfSelectedRules: number) => { }); }; -export const INSTALL_ALL_RULES = (numberOfAllRules: number) => { - return i18n.translate('xpack.securitySolution.siemMigrations.rules.table.installAllRules', { - defaultMessage: - 'Install translated {numberOfAllRules, plural, one {rule} other {rules}} ({numberOfAllRules})', - values: { numberOfAllRules }, - }); +export const INSTALL_TRANSLATED_RULES_EMPTY_STATE = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.table.installTranslatedRulesEmptyState', + { + defaultMessage: 'Install translated rules', + } +); + +export const INSTALL_TRANSLATED_RULES = (numberOfAllRules: number) => { + return i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.table.installTranslatedRules', + { + defaultMessage: + 'Install translated {numberOfAllRules, plural, one {rule} other {rules}} ({numberOfAllRules})', + values: { numberOfAllRules }, + } + ); }; export const INSTALL_SELECTED_ARIA_LABEL = i18n.translate( @@ -57,8 +74,8 @@ export const INSTALL_SELECTED_ARIA_LABEL = i18n.translate( } ); -export const INSTALL_ALL_ARIA_LABEL = i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.table.installAllButtonAriaLabel', +export const INSTALL_TRANSLATED_ARIA_LABEL = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.table.installTranslatedButtonAriaLabel', { defaultMessage: 'Install all translated rules', } diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/index.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/index.tsx index a402e61a444af..4220a35ed4622 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/index.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/index.tsx @@ -12,3 +12,4 @@ export * from './name'; export * from './risk_score'; export * from './severity'; export * from './status'; +export * from './updated'; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/translations.ts index 906e752d79aa0..5b40abd3d7485 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/translations.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/translations.ts @@ -69,3 +69,10 @@ export const COLUMN_SEVERITY = i18n.translate( defaultMessage: 'Severity', } ); + +export const COLUMN_UPDATED = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.tableColumn.updatedLabel', + { + defaultMessage: 'Updated', + } +); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/updated.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/updated.tsx new file mode 100644 index 0000000000000..cec9f86eb7bde --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/updated.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { FormattedRelativePreferenceDate } from '../../../../common/components/formatted_date'; +import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import * as i18n from './translations'; +import type { TableColumn } from './constants'; + +export const createUpdatedColumn = (): TableColumn => { + return { + field: 'updated_at', + name: i18n.COLUMN_UPDATED, + render: (value: RuleMigration['updated_at']) => ( + + ), + sortable: true, + truncateText: false, + align: 'center', + width: '10%', + }; +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/status_badge/index.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/status_badge/index.tsx index 40b3c5ceb5719..171fe0c451826 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/status_badge/index.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/status_badge/index.tsx @@ -30,7 +30,7 @@ const StatusBadgeComponent: React.FC = ({ installedRuleId, 'data-test-subj': dataTestSubj = 'translation-result', }) => { - const translationResult = installedRuleId || !value ? 'full' : value; + const translationResult = installedRuleId ? 'full' : value ?? 'untranslatable'; const displayValue = convertTranslationResultIntoText(translationResult); const color = statusToColorMap[translationResult]; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/translation_details_flyout/translation_tab/index.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/translation_details_flyout/translation_tab/index.tsx index 66836b8ea5631..f2ac76c78434b 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/translation_details_flyout/translation_tab/index.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/translation_details_flyout/translation_tab/index.tsx @@ -24,7 +24,10 @@ import type { RuleMigration } from '../../../../../../common/siem_migrations/mod import { TranslationTabHeader } from './header'; import { RuleQueryComponent } from './rule_query'; import * as i18n from './translations'; -import { convertTranslationResultIntoText } from '../../../utils/helpers'; +import { + convertTranslationResultIntoColor, + convertTranslationResultIntoText, +} from '../../../utils/helpers'; interface TranslationTabProps { ruleMigration: RuleMigration; @@ -66,9 +69,7 @@ export const TranslationTab = ({ ruleMigration }: TranslationTabProps) => { {}} onClickAriaLabel={'Click to update translation status'} > diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/hooks/use_filter_rules_to_install.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/hooks/use_filter_rules_to_install.ts deleted file mode 100644 index f6862d3d90380..0000000000000 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/hooks/use_filter_rules_to_install.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useMemo } from 'react'; -import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen'; -import type { FilterOptions } from '../../../detection_engine/rule_management/logic/types'; - -export type TableFilterOptions = Pick; - -export const useFilterRulesToInstall = ({ - ruleMigrations, - filterOptions, -}: { - ruleMigrations: RuleMigration[]; - filterOptions: TableFilterOptions; -}) => { - const filteredRules = useMemo(() => { - const { filter } = filterOptions; - return ruleMigrations.filter((migration) => { - const name = migration.elastic_rule?.title ?? migration.original_rule.title; - if (filter && !name.toLowerCase().includes(filter.toLowerCase())) { - return false; - } - return true; - }); - }, [filterOptions, ruleMigrations]); - - return filteredRules; -}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/hooks/use_rules_table_columns.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/hooks/use_rules_table_columns.tsx index b7e06b4ea938a..219d2f17de441 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/hooks/use_rules_table_columns.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/hooks/use_rules_table_columns.tsx @@ -14,6 +14,7 @@ import { createRiskScoreColumn, createSeverityColumn, createStatusColumn, + createUpdatedColumn, } from '../components/rules_table_columns'; export const useRulesTableColumns = ({ @@ -27,6 +28,7 @@ export const useRulesTableColumns = ({ }): TableColumn[] => { return useMemo( () => [ + createUpdatedColumn(), createNameColumn({ openMigrationRulePreview }), createStatusColumn(), createRiskScoreColumn(), diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/translations.ts index 23f5a6e3849a0..3b13daa8f0682 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/translations.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/translations.ts @@ -14,6 +14,13 @@ export const GET_MIGRATION_RULES_FAILURE = i18n.translate( } ); +export const GET_MIGRATION_TRANSLATION_STATS_FAILURE = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.getMigrationTranslationStatsFailDescription', + { + defaultMessage: 'Failed to fetch migration translation stats', + } +); + export const INSTALL_MIGRATION_RULES_FAILURE = i18n.translate( 'xpack.securitySolution.siemMigrations.rules.installMigrationRulesFailDescription', { diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_rules.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_rules.ts index 27637daf142ff..92f06b2e37428 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_rules.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_rules.ts @@ -9,10 +9,15 @@ import { useAppToasts } from '../../../common/hooks/use_app_toasts'; import { useGetMigrationRulesQuery } from '../api/hooks/use_get_migration_rules_query'; import * as i18n from './translations'; -export const useGetMigrationRules = (migrationId: string) => { +export const useGetMigrationRules = (params: { + migrationId: string; + page?: number; + perPage?: number; + searchTerm?: string; +}) => { const { addError } = useAppToasts(); - return useGetMigrationRulesQuery(migrationId, { + return useGetMigrationRulesQuery(params, { onError: (error) => { addError(error, { title: i18n.GET_MIGRATION_RULES_FAILURE }); }, diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_translation_stats.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_translation_stats.ts new file mode 100644 index 0000000000000..081876ba266a9 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_translation_stats.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useAppToasts } from '../../../common/hooks/use_app_toasts'; +import { useGetMigrationTranslationStatsQuery } from '../api/hooks/use_get_migration_translation_stats_query'; +import * as i18n from './translations'; + +export const useGetMigrationTranslationStats = (migrationId: string) => { + const { addError } = useAppToasts(); + + return useGetMigrationTranslationStatsQuery(migrationId, { + onError: (error) => { + addError(error, { title: i18n.GET_MIGRATION_TRANSLATION_STATS_FAILURE }); + }, + }); +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_install_all_migration_rules.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_install_translated_migration_rules.ts similarity index 65% rename from x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_install_all_migration_rules.ts rename to x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_install_translated_migration_rules.ts index 105ea651d0a8c..67ee3f099aca0 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_install_all_migration_rules.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/logic/use_install_translated_migration_rules.ts @@ -6,13 +6,13 @@ */ import { useAppToasts } from '../../../common/hooks/use_app_toasts'; -import { useInstallAllMigrationRulesMutation } from '../api/hooks/use_install_all_migration_rules_mutation'; +import { useInstallTranslatedMigrationRulesMutation } from '../api/hooks/use_install_translated_migration_rules_mutation'; import * as i18n from './translations'; -export const useInstallAllMigrationRules = (migrationId: string) => { +export const useInstallTranslatedMigrationRules = (migrationId: string) => { const { addError } = useAppToasts(); - return useInstallAllMigrationRulesMutation(migrationId, { + return useInstallTranslatedMigrationRulesMutation(migrationId, { onError: (error) => { addError(error, { title: i18n.INSTALL_MIGRATION_RULES_FAILURE }); }, diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/utils/helpers.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/utils/helpers.tsx index cd49311db21eb..fe3fbf9945077 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/utils/helpers.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/utils/helpers.tsx @@ -11,6 +11,22 @@ import { } from '../../../../common/siem_migrations/model/rule_migration.gen'; import * as i18n from './translations'; +export const convertTranslationResultIntoColor = (status?: RuleMigrationTranslationResult) => { + switch (status) { + case RuleMigrationTranslationResultEnum.full: + return 'primary'; + + case RuleMigrationTranslationResultEnum.partial: + return 'warning'; + + case RuleMigrationTranslationResultEnum.untranslatable: + return 'danger'; + + default: + throw new Error(i18n.SIEM_TRANSLATION_RESULT_UNKNOWN_ERROR(status)); + } +}; + export const convertTranslationResultIntoText = (status?: RuleMigrationTranslationResult) => { switch (status) { case RuleMigrationTranslationResultEnum.full: @@ -23,6 +39,6 @@ export const convertTranslationResultIntoText = (status?: RuleMigrationTranslati return i18n.SIEM_TRANSLATION_RESULT_UNTRANSLATABLE_LABEL; default: - return i18n.SIEM_TRANSLATION_RESULT_UNKNOWN_LABEL; + throw new Error(i18n.SIEM_TRANSLATION_RESULT_UNKNOWN_ERROR(status)); } }; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/utils/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/utils/translations.ts index bc098936c00f7..366ad435c61b4 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/utils/translations.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/utils/translations.ts @@ -34,3 +34,13 @@ export const SIEM_TRANSLATION_RESULT_UNKNOWN_LABEL = i18n.translate( defaultMessage: 'Unknown', } ); + +export const SIEM_TRANSLATION_RESULT_UNKNOWN_ERROR = (status?: string) => { + return i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.translationResult.unknownError', + { + defaultMessage: 'Unknown translation result status: ({status})', + values: { status }, + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts index 0d880484877f6..b9645a3de374e 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts @@ -9,6 +9,7 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { GetRuleMigrationRequestParams, + GetRuleMigrationRequestQuery, type GetRuleMigrationResponse, } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATION_PATH } from '../../../../../common/siem_migrations/constants'; @@ -29,18 +30,35 @@ export const registerSiemRuleMigrationsGetRoute = ( { version: '1', validate: { - request: { params: buildRouteValidationWithZod(GetRuleMigrationRequestParams) }, + request: { + params: buildRouteValidationWithZod(GetRuleMigrationRequestParams), + query: buildRouteValidationWithZod(GetRuleMigrationRequestQuery), + }, }, }, withLicense(async (context, req, res): Promise> => { - const migrationId = req.params.migration_id; + const { migration_id: migrationId } = req.params; + const { page, per_page: perPage, search_term: searchTerm } = req.query; try { const ctx = await context.resolve(['securitySolution']); const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const migrationRules = await ruleMigrationsClient.data.rules.get({ migrationId }); + let from = 0; + if (page && perPage) { + from = page * perPage; + } + const size = perPage; - return res.ok({ body: migrationRules }); + const result = await ruleMigrationsClient.data.rules.get( + { + migrationId, + searchTerm, + }, + from, + size + ); + + return res.ok({ body: result }); } catch (err) { logger.error(err); return res.badRequest({ body: err.message }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts index 601b156aee040..c6f3c51a1bb53 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts @@ -12,13 +12,14 @@ import { registerSiemRuleMigrationsUpdateRoute } from './update'; import { registerSiemRuleMigrationsGetRoute } from './get'; import { registerSiemRuleMigrationsStartRoute } from './start'; import { registerSiemRuleMigrationsStatsRoute } from './stats'; +import { registerSiemRuleMigrationsTranslationStatsRoute } from './translation_stats'; import { registerSiemRuleMigrationsStopRoute } from './stop'; import { registerSiemRuleMigrationsStatsAllRoute } from './stats_all'; import { registerSiemRuleMigrationsResourceUpsertRoute } from './resources/upsert'; import { registerSiemRuleMigrationsResourceGetRoute } from './resources/get'; import { registerSiemRuleMigrationsRetryRoute } from './retry'; -import { registerSiemRuleMigrationsInstallRoute } from './rules/install'; -import { registerSiemRuleMigrationsInstallTranslatedRoute } from './rules/install_translated'; +import { registerSiemRuleMigrationsInstallRoute } from './install'; +import { registerSiemRuleMigrationsInstallTranslatedRoute } from './install_translated'; export const registerSiemRuleMigrationsRoutes = ( router: SecuritySolutionPluginRouter, @@ -31,6 +32,7 @@ export const registerSiemRuleMigrationsRoutes = ( registerSiemRuleMigrationsStartRoute(router, logger); registerSiemRuleMigrationsRetryRoute(router, logger); registerSiemRuleMigrationsStatsRoute(router, logger); + registerSiemRuleMigrationsTranslationStatsRoute(router, logger); registerSiemRuleMigrationsStopRoute(router, logger); registerSiemRuleMigrationsInstallRoute(router, logger); registerSiemRuleMigrationsInstallTranslatedRoute(router, logger); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/install.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/install.ts similarity index 85% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/install.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/install.ts index 7e8b1a25a4837..7b41ea536aadf 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/install.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/install.ts @@ -7,15 +7,15 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import { SIEM_RULE_MIGRATION_INSTALL_PATH } from '../../../../../../common/siem_migrations/constants'; -import type { InstallMigrationRulesResponse } from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_INSTALL_PATH } from '../../../../../common/siem_migrations/constants'; +import type { InstallMigrationRulesResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { InstallMigrationRulesRequestBody, InstallMigrationRulesRequestParams, -} from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; -import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import { withLicense } from '../util/with_license'; -import { installTranslated } from '../util/installation'; +} from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; +import { installTranslated } from './util/installation'; export const registerSiemRuleMigrationsInstallRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/install_translated.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/install_translated.ts similarity index 84% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/install_translated.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/install_translated.ts index 6fd4b5f0980b6..ac6a598c4b92f 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/install_translated.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/install_translated.ts @@ -7,12 +7,12 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import { SIEM_RULE_MIGRATION_INSTALL_TRANSLATED_PATH } from '../../../../../../common/siem_migrations/constants'; -import type { InstallTranslatedMigrationRulesResponse } from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; -import { InstallTranslatedMigrationRulesRequestParams } from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; -import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import { withLicense } from '../util/with_license'; -import { installTranslated } from '../util/installation'; +import { SIEM_RULE_MIGRATION_INSTALL_TRANSLATED_PATH } from '../../../../../common/siem_migrations/constants'; +import type { InstallTranslatedMigrationRulesResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { InstallTranslatedMigrationRulesRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; +import { installTranslated } from './util/installation'; export const registerSiemRuleMigrationsInstallTranslatedRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/translation_stats.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/translation_stats.ts new file mode 100644 index 0000000000000..4f9d12385e32d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/translation_stats.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { GetRuleMigrationTranslationStatsResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { GetRuleMigrationTranslationStatsRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; + +export const registerSiemRuleMigrationsTranslationStatsRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: buildRouteValidationWithZod(GetRuleMigrationTranslationStatsRequestParams), + }, + }, + }, + withLicense( + async ( + context, + req, + res + ): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const stats = await ruleMigrationsClient.data.rules.getTranslationStats(migrationId); + + return res.ok({ body: stats }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/util/installation.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/util/installation.ts index ee211e8a935de..756b4b99612c7 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/util/installation.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/util/installation.ts @@ -177,7 +177,7 @@ export const installTranslated = async ({ const detectionRulesClient = securitySolutionContext.getDetectionRulesClient(); const ruleMigrationsClient = securitySolutionContext.getSiemRuleMigrationsClient(); - const rulesToInstall = await ruleMigrationsClient.data.rules.get({ + const { data: rulesToInstall } = await ruleMigrationsClient.data.rules.get({ migrationId, ids, installable: true, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts index 0a82e2c311906..209f2e4416e16 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts @@ -15,11 +15,15 @@ import type { QueryDslQueryContainer, } from '@elastic/elasticsearch/lib/api/types'; import type { StoredRuleMigration } from '../types'; -import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; +import { + SiemMigrationRuleTranslationResult, + SiemMigrationStatus, +} from '../../../../../common/siem_migrations/constants'; import type { ElasticRule, RuleMigration, RuleMigrationTaskStats, + RuleMigrationTranslationStats, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client'; @@ -39,6 +43,7 @@ export interface RuleMigrationFilterOptions { status?: SiemMigrationStatus | SiemMigrationStatus[]; ids?: string[]; installable?: boolean; + searchTerm?: string; } /* BULK_MAX_SIZE defines the number to break down the bulk operations by. @@ -46,6 +51,20 @@ export interface RuleMigrationFilterOptions { */ const BULK_MAX_SIZE = 500 as const; +const getInstallableConditions = (): QueryDslQueryContainer[] => { + return [ + { term: { translation_result: SiemMigrationRuleTranslationResult.FULL } }, + { + nested: { + path: 'elastic_rule', + query: { + bool: { must_not: { exists: { field: 'elastic_rule.id' } } }, + }, + }, + }, + ]; +}; + export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient { /** Indexes an array of rule migrations to be processed */ async create(ruleMigrations: CreateRuleMigrationInput[]): Promise { @@ -108,18 +127,24 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient } /** Retrieves an array of rule documents of a specific migrations */ - async get(filters: RuleMigrationFilterOptions): Promise { + async get( + filters: RuleMigrationFilterOptions, + from?: number, + size?: number + ): Promise<{ total: number; data: StoredRuleMigration[] }> { const index = await this.getIndexName(); const query = this.getFilterQuery(filters); - const storedRuleMigrations = await this.esClient - .search({ index, query, sort: '_doc' }) - .then(this.processResponseHits.bind(this)) + const result = await this.esClient + .search({ index, query, sort: '_doc', from, size }) .catch((error) => { this.logger.error(`Error searching rule migrations: ${error.message}`); throw error; }); - return storedRuleMigrations; + return { + total: this.getTotalHits(result), + data: this.processResponseHits(result), + }; } /** @@ -217,6 +242,49 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient }); } + /** Retrieves the translation stats for the rule migrations with the provided id */ + async getTranslationStats(migrationId: string): Promise { + const index = await this.getIndexName(); + const query = this.getFilterQuery({ migrationId }); + + const aggregations = { + prebuilt: { + filter: { + nested: { + path: 'elastic_rule', + query: { exists: { field: 'elastic_rule.prebuilt_rule_id' } }, + }, + }, + }, + installable: { + filter: { + bool: { + must: getInstallableConditions(), + }, + }, + }, + }; + const result = await this.esClient + .search({ index, query, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting rule migrations stats: ${error.message}`); + throw error; + }); + + const bucket = result.aggregations ?? {}; + const total = this.getTotalHits(result); + const prebuilt = (bucket.prebuilt as AggregationsFilterAggregate)?.doc_count ?? 0; + return { + id: migrationId, + rules: { + total, + prebuilt, + custom: total - prebuilt, + installable: (bucket.installable as AggregationsFilterAggregate)?.doc_count ?? 0, + }, + }; + } + /** Retrieves the stats for the rule migrations with the provided id */ async getStats(migrationId: string): Promise { const index = await this.getIndexName(); @@ -295,6 +363,7 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient status, ids, installable, + searchTerm, }: RuleMigrationFilterOptions): QueryDslQueryContainer { const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; if (status) { @@ -308,15 +377,15 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient filter.push({ terms: { _id: ids } }); } if (installable) { - filter.push( - { term: { translation_result: 'full' } }, - { - nested: { - path: 'elastic_rule', - query: { bool: { must_not: { exists: { field: 'elastic_rule.id' } } } }, - }, - } - ); + filter.push(...getInstallableConditions()); + } + if (searchTerm?.length) { + filter.push({ + nested: { + path: 'elastic_rule', + query: { match: { 'elastic_rule.title': searchTerm } }, + }, + }); } return { bool: { filter } }; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts index 8e8a3c5ee0f27..09a4bef34c279 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts @@ -19,17 +19,17 @@ export const ruleMigrationsFieldMap: FieldMap Date: Tue, 3 Dec 2024 19:23:13 +0100 Subject: [PATCH 22/51] [Infra] Exclude frozen/cold data tiers from source queries (#201804) Closes #201568 ## Summary Adds the exclude data tiers settings to the `/api/metrics/source/hasData` and `/api/metrics/source/{sourceId}` requests. Also applies it to the `getIndexStatus` API call. --------- Co-authored-by: Elastic Machine Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../server/lib/helpers/tier_filter.ts | 4 +- .../elasticsearch_source_status_adapter.ts | 16 +- .../helpers/get_infra_metrics_client.test.ts | 151 ++++++++++++++++++ .../lib/helpers/get_infra_metrics_client.ts | 27 ++++ .../server/routes/metrics_sources/index.ts | 2 +- .../infra/tsconfig.json | 3 +- .../observability/server/ui_settings.ts | 6 +- 7 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index c1f8d5e3fce1f..83171dea7f444 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -13,10 +13,10 @@ export function getDataTierFilterCombined({ excludedDataTiers, }: { filter?: QueryDslQueryContainer; - excludedDataTiers: DataTier[]; + excludedDataTiers?: DataTier[]; }): QueryDslQueryContainer | undefined { if (!filter) { - return excludedDataTiers.length > 0 ? excludeTiersQuery(excludedDataTiers)[0] : undefined; + return excludedDataTiers?.length ? excludeTiersQuery(excludedDataTiers)[0] : undefined; } return !excludedDataTiers diff --git a/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts b/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts index 54e2831899482..ff8058011baea 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts @@ -5,11 +5,14 @@ * 2.0. */ +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; import type { InfraPluginRequestHandlerContext } from '../../../types'; import { isNoSuchRemoteClusterMessage, NoSuchRemoteClusterError } from '../../sources/errors'; -import { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status'; -import { InfraDatabaseGetIndicesResponse } from '../framework'; -import { KibanaFramework } from '../framework/kibana_framework_adapter'; +import type { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status'; +import type { InfraDatabaseGetIndicesResponse } from '../framework'; +import type { KibanaFramework } from '../framework/kibana_framework_adapter'; export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusAdapter { constructor(private readonly framework: KibanaFramework) {} @@ -46,6 +49,12 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA requestContext: InfraPluginRequestHandlerContext, indexNames: string ): Promise { + const { uiSettings } = await requestContext.core; + + const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); + + const filter = excludedDataTiers.length ? excludeTiersQuery(excludedDataTiers) : []; + return await this.framework .callWithRequest(requestContext, 'search', { ignore_unavailable: true, @@ -54,6 +63,7 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA size: 0, terminate_after: 1, track_total_hits: 1, + query: { bool: { filter } }, }) .then( (response) => { diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts new file mode 100644 index 0000000000000..3eb8c47c274d9 --- /dev/null +++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest } from '@kbn/core-http-server'; +import { DataTier } from '@kbn/observability-shared-plugin/common'; +import { InfraBackendLibs } from '../infra_types'; +import { getInfraMetricsClient } from './get_infra_metrics_client'; +import { InfraPluginRequestHandlerContext } from '../../types'; +import { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; + +const withExcludedDataTiers = (tiers: DataTier[]) => ({ + uiSettings: { + client: { + get: () => Promise.resolve(tiers), + }, + }, +}); + +const mockedInfra = { getMetricsIndices: () => Promise.resolve(['*.indices']) }; + +const infraMetricsTestHarness = + (tiers: DataTier[], input: QueryDslQueryContainer | undefined, output: QueryDslQueryContainer) => + async () => { + const callWithRequest = jest.fn(); + + const mockedCore = withExcludedDataTiers(tiers); + + const context = { + infra: Promise.resolve(mockedInfra), + core: Promise.resolve(mockedCore), + } as unknown as InfraPluginRequestHandlerContext; + + const client = await getInfraMetricsClient({ + libs: { framework: { callWithRequest } } as unknown as InfraBackendLibs, + context, + request: {} as unknown as KibanaRequest, + }); + + await client.search({ + body: { + query: input, + size: 1, + track_total_hits: false, + }, + }); + + expect(callWithRequest).toBeCalledWith( + context, + 'search', + { + body: { + query: output, + size: 1, + track_total_hits: false, + }, + ignore_unavailable: true, + index: ['*.indices'], + }, + {} + ); + }; + +describe('getInfraMetricsClient', () => { + it( + 'defines an empty must_not query if given no data tiers to filter by', + infraMetricsTestHarness([], undefined, { bool: { must_not: [] } }) + ); + + it( + 'includes excluded data tiers in the request filter by default', + infraMetricsTestHarness(['data_frozen'], undefined, { + bool: { + must_not: [ + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], + }, + }) + ); + + it( + 'merges provided filters with the excluded data tier filter', + infraMetricsTestHarness( + ['data_frozen'], + { + bool: { + must_not: { + exists: { + field: 'a-field', + }, + }, + }, + }, + { + bool: { + must_not: [ + { + exists: { + field: 'a-field', + }, + }, + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], + }, + } + ) + ); + + it( + 'merges other query params with the excluded data tiers filter', + infraMetricsTestHarness( + ['data_frozen'], + { + bool: { + must: { + exists: { + field: 'a-field', + }, + }, + }, + }, + { + bool: { + must: { + exists: { + field: 'a-field', + }, + }, + must_not: [ + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], + }, + } + ) + ); +}); diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts index d560ffc358a5c..96ae89b902285 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts @@ -6,6 +6,10 @@ */ import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import type { KibanaRequest } from '@kbn/core/server'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import type { InfraPluginRequestHandlerContext } from '../../types'; import type { InfraBackendLibs } from '../infra_types'; @@ -29,12 +33,25 @@ export async function getInfraMetricsClient({ }) { const { framework } = libs; const infraContext = await context.infra; + const { uiSettings } = await context.core; + + const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); const metricsIndices = await infraContext.getMetricsIndices(); + const excludedQuery = excludedDataTiers.length + ? excludeTiersQuery(excludedDataTiers)[0].bool!.must_not! + : []; + return { search( searchParams: TParams ): Promise> { + const searchFilter = searchParams.body.query?.bool?.must_not ?? []; + + // This flattens arrays by one level, and non-array values can be added as well, so it all + // results in a nice [QueryDsl, QueryDsl, ...] array. + const mustNot = ([] as QueryDslQueryContainer[]).concat(searchFilter, excludedQuery); + return framework.callWithRequest( context, 'search', @@ -42,6 +59,16 @@ export async function getInfraMetricsClient({ ...searchParams, ignore_unavailable: true, index: metricsIndices, + body: { + ...searchParams.body, + query: { + ...searchParams.body.query, + bool: { + ...searchParams.body.query?.bool, + must_not: mustNot, + }, + }, + }, }, request ) as Promise; diff --git a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts index 20dcbc1b7e20b..77bdee365e3bf 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts @@ -208,7 +208,7 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => }, async (context, request, response) => { try { - const modules = castArray(request.query.modules); + const modules = request.query.modules ? castArray(request.query.modules) : []; if (modules.length > MAX_MODULES) { throw Boom.badRequest( diff --git a/x-pack/plugins/observability_solution/infra/tsconfig.json b/x-pack/plugins/observability_solution/infra/tsconfig.json index efd8be77b688c..1e130261d5346 100644 --- a/x-pack/plugins/observability_solution/infra/tsconfig.json +++ b/x-pack/plugins/observability_solution/infra/tsconfig.json @@ -116,7 +116,8 @@ "@kbn/entityManager-plugin", "@kbn/entities-schema", "@kbn/zod", - "@kbn/observability-utils-server" + "@kbn/observability-utils-server", + "@kbn/observability-utils-common" ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts index 1a387f24fbaed..e5695a3c9034d 100644 --- a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts +++ b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts @@ -504,7 +504,7 @@ export const uiSettings: Record = { value: 1.7, description: i18n.translate('xpack.observability.profilingDatacenterPUEUiSettingDescription', { defaultMessage: `Data center power usage effectiveness (PUE) measures how efficiently a data center uses energy. Defaults to 1.7, the average on-premise data center PUE according to the Uptime Institute survey - + You can also use the PUE that corresponds with your cloud provider: '