Skip to content

Commit

Permalink
[Defend Workflows] Fix artifact entries list FTR (#189961)
Browse files Browse the repository at this point in the history 
## Summary

**tl;wr;**
- instead of selecting from dropdown, values are entered for Event
Filters
- artifacts for update/delete test cases are created via API call
instead of UI interaction

> [!note]
> FTRs needs to be re-enabled before merge, revert this
e0858a5 - done ✅

---

This PR attempts to decrease flakiness in `artifact_entries_list` FTR.
Looks like the reason for flakiness was the dropdown for the field to be
filtered. After spending a bit of a time looking at the code, couldn't
find any quick solution why it can be flaky, so instead just tried to
improve the tests. For that, added a 'special' (aka hack) test case to
just try to set event filter 100 times, and also a failing test case
just to receive logs (77455d7) - that's
why all flaky runners are red, you need to see the logs.

It didn't work to add some waits to let React re-render and let the API
call for suggestions be finished, didn't work to select from dropdown by
text instead of test id, didn't work to add even more wait in between
multiple user actions. See logs here and search for `results are in`
inside the logs:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6744#019145e0-445d-4978-89c8-701908e14e3f
<img width="388" alt="image"
src="https://github.com/user-attachments/assets/be45d549-8902-4e8d-926e-2e575eca0971">

In the end, looks like the solution is simply `input` the values instead
of selecting them from the dropdown. In the last hacky-flaky run, it
passed all 100 times:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6756
<img width="360" alt="image"
src="https://github.com/user-attachments/assets/b46b7405-0621-4bf7-af88-a39309298b6b">

And a last flaky run without the hack, with the official flaky runner:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6772
100/100 ✅

Also, entries for the update/delete test cases are created via API call
instead of UI interaction, so those cases overlap less with the create
test case.

---------

Co-authored-by: Elastic Machine <[email protected]>
  • Loading branch information
@gergoabraham @elasticmachine
gergoabraham and elasticmachine authored Aug 27, 2024
1 parent 51e76d8 commit 8e9b827
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 20 deletions.
23 changes: 10 additions & 13 deletions x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const retry = getService('retry');
const esClient = getService('es');
const supertest = getService('supertest');
const find = getService('find');
const toasts = getService('toasts');
const policyTestResources = getService('policyTestResources');
const unzipPromisify = promisify(unzip);
Expand All @@ -52,12 +51,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
.set('kbn-xsrf', 'true');
};

// Failing: See https://github.com/elastic/kibana/issues/187314
// Failing: See https://github.com/elastic/kibana/issues/187383
// Failing: See https://github.com/elastic/kibana/issues/188131
// Failing: See https://github.com/elastic/kibana/issues/188125
describe.skip('For each artifact list under management', function () {
targetTags(this, ['@ess', '@skipInServerless']);
describe('For each artifact list under management', function () {
targetTags(this, ['@ess', '@serverless']);
this.timeout(60_000 * 5);

let indexedData: IndexedHostsAndAlertsResponse;
Expand Down Expand Up @@ -154,9 +149,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
suffix?: string
) => {
for (const formAction of actions) {
if (formAction.type === 'customClick') {
await find.clickByCssSelector(formAction.selector, testSubjects.FIND_TIME);
} else if (formAction.type === 'click') {
if (formAction.type === 'click') {
await testSubjects.click(formAction.selector);
} else if (formAction.type === 'input') {
const newValue = (formAction.value || '') + (suffix ? suffix : '');
Expand Down Expand Up @@ -265,7 +258,9 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
});

it(`should be able to update an existing ${testData.title} entry`, async () => {
await createArtifact(testData);
await endpointArtifactsTestResources.createArtifact(testData.listId, testData.createBody);
await browser.refresh();

await updateArtifact(testData, { policyId: policyInfo.packagePolicy.id });

// Check edited artifact is in the list with new values (wait for list to be updated)
Expand Down Expand Up @@ -299,7 +294,9 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
});

it(`should be able to delete the existing ${testData.title} entry`, async () => {
await createArtifact(testData);
await endpointArtifactsTestResources.createArtifact(testData.listId, testData.createBody);
await browser.refresh();

await deleteArtifact(testData);
// We only expect one artifact to have been visible
await testSubjects.missingOrFail(testData.delete.card);
Expand Down Expand Up @@ -336,7 +333,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
});

const testData = getCreateMultipleData();
it(`should get correct atifact when multiple entries are created`, async () => {
it(`should get correct artifact when multiple entries are created`, async () => {
// Create first trusted app
await createArtifact(testData, {
policyId: firstPolicy.packagePolicy.id,
Expand Down
76 changes: 70 additions & 6 deletions x-pack/test/security_solution_endpoint/apps/integrations/mocks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@

import { FullAgentPolicy } from '@kbn/fleet-plugin/common/types';
import { ArtifactElasticsearchProperties } from '@kbn/fleet-plugin/server/services/artifacts/types';
import { GLOBAL_ARTIFACT_TAG } from '@kbn/security-solution-plugin/common/endpoint/service/artifacts';
import { InternalUnifiedManifestBaseSchema } from '@kbn/security-solution-plugin/server/endpoint/schemas/artifacts';
import { TranslatedExceptionListItem } from '@kbn/security-solution-plugin/server/endpoint/schemas/artifacts/lists';
import { CreateExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';

export interface AgentPolicyResponseType {
_index: string;
Expand Down Expand Up @@ -119,6 +122,19 @@ export const getArtifactsListTestsData = () => [
confirmSelector: 'trustedAppsListPage-deleteModal-submitButton',
card: 'trustedAppsListPage-card',
},
listId: ENDPOINT_ARTIFACT_LISTS.trustedApps.id,
createBody: {
entries: [
{
type: 'match',
field: 'process.hash.sha256',
value: 'a4370c0cf81686c0b696fa6261c9d3e0d810ae704ab8301839dffd5d5112f476',
operator: 'included',
},
],
tags: [GLOBAL_ARTIFACT_TAG],
os_types: ['windows'],
} as Partial<CreateExceptionListItemSchema>,
urlPath: 'trusted_apps',
pageObject: 'trustedApps',
fleetArtifact: {
Expand Down Expand Up @@ -206,8 +222,9 @@ export const getArtifactsListTestsData = () => [
selector: 'fieldAutocompleteComboBox',
},
{
type: 'customClick',
selector: 'button[title="agent.ephemeral_id"]',
type: 'input',
selector: 'fieldAutocompleteComboBox',
value: 'agent.ephemeral_id',
},
{
type: 'click',
Expand Down Expand Up @@ -247,10 +264,6 @@ export const getArtifactsListTestsData = () => [
selector: 'fieldAutocompleteComboBox',
value: 'agent.id',
},
{
type: 'customClick',
selector: 'button[title="agent.id"]',
},
{
type: 'input',
selector: 'valuesAutocompleteMatch',
Expand Down Expand Up @@ -281,6 +294,19 @@ export const getArtifactsListTestsData = () => [
confirmSelector: 'EventFiltersListPage-deleteModal-submitButton',
card: 'EventFiltersListPage-card',
},
listId: ENDPOINT_ARTIFACT_LISTS.eventFilters.id,
createBody: {
entries: [
{
field: 'agent.ephemeral_id',
value: 'endpoint',
type: 'match',
operator: 'included',
},
],
tags: [GLOBAL_ARTIFACT_TAG],
os_types: ['windows'],
} as Partial<CreateExceptionListItemSchema>,
urlPath: 'event_filters',
pageObject: 'eventFilters',
fleetArtifact: {
Expand Down Expand Up @@ -456,6 +482,31 @@ export const getArtifactsListTestsData = () => [
card: 'blocklistCard',
},
pageObject: 'blocklist',
listId: ENDPOINT_ARTIFACT_LISTS.blocklists.id,
createBody: {
entries: [
{
type: 'match_any',
field: 'file.hash.md5',
value: ['741462ab431a22233c787baab9b653c7'],
operator: 'included',
},
{
type: 'match_any',
field: 'file.hash.sha1',
value: ['aedb279e378bed6c2db3c9dc9e12ba635e0b391c'],
operator: 'included',
},
{
type: 'match_any',
field: 'file.hash.sha256',
value: ['a4370c0cf81686c0b696fa6261c9d3e0d810ae704ab8301839dffd5d5112f476'],
operator: 'included',
},
],
tags: [GLOBAL_ARTIFACT_TAG],
os_types: ['windows'],
} as Partial<CreateExceptionListItemSchema>,
urlPath: 'blocklist',
fleetArtifact: {
identifier: 'endpoint-blocklist-windows-v1',
Expand Down Expand Up @@ -610,6 +661,19 @@ export const getArtifactsListTestsData = () => [
confirmSelector: 'hostIsolationExceptionsDeletionConfirm',
card: 'hostIsolationExceptionsCard',
},
listId: ENDPOINT_ARTIFACT_LISTS.hostIsolationExceptions.id,
createBody: {
entries: [
{
type: 'match',
field: 'destination.ip',
value: '1.1.1.1',
operator: 'included',
},
],
tags: [GLOBAL_ARTIFACT_TAG],
os_types: ['windows', 'linux', 'macos'],
} as Partial<CreateExceptionListItemSchema>,
pageObject: 'hostIsolationExceptions',
urlPath: 'host_isolation_exceptions',
fleetArtifact: {
Expand Down
27 changes: 26 additions & 1 deletion x-pack/test/security_solution_endpoint/services/endpoint_artifacts.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,12 @@ import type {
CreateExceptionListSchema,
ExceptionListItemSchema,
} from '@kbn/securitysolution-io-ts-list-types';
import { EXCEPTION_LIST_ITEM_URL, EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants';
import {
ENDPOINT_ARTIFACT_LISTS,
ENDPOINT_ARTIFACT_LIST_IDS,
EXCEPTION_LIST_ITEM_URL,
EXCEPTION_LIST_URL,
} from '@kbn/securitysolution-list-constants';
import { Response } from 'superagent';
import { ExceptionsListItemGenerator } from '@kbn/security-solution-plugin/common/endpoint/data_generators/exceptions_list_item_generator';
import { TRUSTED_APPS_EXCEPTION_LIST_DEFINITION } from '@kbn/security-solution-plugin/public/management/pages/trusted_apps/constants';
Expand Down Expand Up @@ -122,6 +127,26 @@ export class EndpointArtifactsTestResources extends FtrService {
return this.createExceptionItem(blocklist);
}

async createArtifact(
listId: (typeof ENDPOINT_ARTIFACT_LIST_IDS)[number],
overrides: Partial<CreateExceptionListItemSchema> = {}
): Promise<ArtifactTestData | undefined> {
switch (listId) {
case ENDPOINT_ARTIFACT_LISTS.trustedApps.id: {
return this.createTrustedApp(overrides);
}
case ENDPOINT_ARTIFACT_LISTS.eventFilters.id: {
return this.createEventFilter(overrides);
}
case ENDPOINT_ARTIFACT_LISTS.blocklists.id: {
return this.createBlocklist(overrides);
}
case ENDPOINT_ARTIFACT_LISTS.hostIsolationExceptions.id: {
return this.createHostIsolationException(overrides);
}
}
}

async getArtifactsFromUnifiedManifestSO(): Promise<
Array<
InternalUnifiedManifestSchemaResponseType['_source']['endpoint:unified-user-artifact-manifest']
Expand Down

0 comments on commit 8e9b827

Please sign in to comment.