Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.10] [Security Solution] Fix value lists tests flakiness (#164253) #164324

Merged
merged 1 commit into from
Aug 21, 2023
Merged

[8.10] [Security Solution] Fix value lists tests flakiness (#164253) #164324

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 77 additions & 49 deletions ...st/security_solution_cypress/cypress/e2e/detection_response/value_lists/value_lists.cy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,33 +17,35 @@ import {
selectValueListsFile,
uploadValueList,
selectValueListType,
deleteAllValueListsFromUI,
closeValueListsModal,
importValueList,
deleteValueListsFile,
exportValueList,
waitForListsIndex,
deleteValueLists,
} from '../../../tasks/lists';
import {
VALUE_LISTS_TABLE,
VALUE_LISTS_ROW,
VALUE_LISTS_MODAL_ACTIVATOR,
} from '../../../screens/lists';
import { refreshIndex } from '../../../tasks/api_calls/elasticsearch';

const TEXT_LIST_FILE_NAME = 'value_list.txt';
const IPS_LIST_FILE_NAME = 'ip_list.txt';
const CIDRS_LIST_FILE_NAME = 'cidr_list.txt';

describe('value lists', () => {
describe('management modal', { tags: [tag.ESS, tag.SERVERLESS] }, () => {
beforeEach(() => {
login();
deleteValueLists([TEXT_LIST_FILE_NAME, IPS_LIST_FILE_NAME, CIDRS_LIST_FILE_NAME]);
createListsIndex();
visitWithoutDateRange(DETECTIONS_RULE_MANAGEMENT_URL);
waitForListsIndex();
waitForValueListsModalToBeLoaded();
});

afterEach(() => {
deleteAllValueListsFromUI();
});

it('can open and close the modal', () => {
openValueListsModal();
closeValueListsModal();
Expand All @@ -55,121 +57,122 @@ describe('value lists', () => {
});

it('creates a "keyword" list from an uploaded file', () => {
const listName = 'value_list.txt';
selectValueListType('keyword');
selectValueListsFile(listName);
selectValueListsFile(TEXT_LIST_FILE_NAME);
uploadValueList();

cy.get(VALUE_LISTS_TABLE)
.find(VALUE_LISTS_ROW)
.should(($row) => {
expect($row.text()).to.contain(listName);
expect($row.text()).to.contain(TEXT_LIST_FILE_NAME);
expect($row.text()).to.contain('Keywords');
});
});

it('creates a "text" list from an uploaded file', () => {
const listName = 'value_list.txt';
selectValueListType('text');
selectValueListsFile(listName);
selectValueListsFile(TEXT_LIST_FILE_NAME);
uploadValueList();

cy.get(VALUE_LISTS_TABLE)
.find(VALUE_LISTS_ROW)
.should(($row) => {
expect($row.text()).to.contain(listName);
expect($row.text()).to.contain(TEXT_LIST_FILE_NAME);
expect($row.text()).to.contain('Text');
});
});

it('creates a "ip" list from an uploaded file', () => {
const listName = 'ip_list.txt';
selectValueListType('ip');
selectValueListsFile(listName);
selectValueListsFile(IPS_LIST_FILE_NAME);
uploadValueList();

cy.get(VALUE_LISTS_TABLE)
.find(VALUE_LISTS_ROW)
.should(($row) => {
expect($row.text()).to.contain(listName);
expect($row.text()).to.contain(IPS_LIST_FILE_NAME);
expect($row.text()).to.contain('IP addresses');
});
});

it('creates a "ip_range" list from an uploaded file', () => {
const listName = 'cidr_list.txt';
selectValueListType('ip_range');
selectValueListsFile(listName);
selectValueListsFile(CIDRS_LIST_FILE_NAME);
uploadValueList();

cy.get(VALUE_LISTS_TABLE)
.find(VALUE_LISTS_ROW)
.should(($row) => {
expect($row.text()).to.contain(listName);
expect($row.text()).to.contain(CIDRS_LIST_FILE_NAME);
expect($row.text()).to.contain('IP ranges');
});
});
});

describe('delete list types', () => {
it('deletes a "keyword" list from an uploaded file', () => {
const listName = 'value_list.txt';
importValueList(listName, 'keyword');
importValueList(TEXT_LIST_FILE_NAME, 'keyword');
openValueListsModal();
deleteValueListsFile(listName);
deleteValueListsFile(TEXT_LIST_FILE_NAME);
cy.get(VALUE_LISTS_TABLE)
.find(VALUE_LISTS_ROW)
.should(($row) => {
expect($row.text()).not.to.contain(listName);
expect($row.text()).not.to.contain(TEXT_LIST_FILE_NAME);
});
});

it('deletes a "text" list from an uploaded file', () => {
const listName = 'value_list.txt';
importValueList(listName, 'text');
importValueList(TEXT_LIST_FILE_NAME, 'text');
openValueListsModal();
deleteValueListsFile(listName);
deleteValueListsFile(TEXT_LIST_FILE_NAME);
cy.get(VALUE_LISTS_TABLE)
.find(VALUE_LISTS_ROW)
.should(($row) => {
expect($row.text()).not.to.contain(listName);
expect($row.text()).not.to.contain(TEXT_LIST_FILE_NAME);
});
});

it('deletes a "ip" from an uploaded file', () => {
const listName = 'ip_list.txt';
importValueList(listName, 'ip');
importValueList(IPS_LIST_FILE_NAME, 'ip');
openValueListsModal();
deleteValueListsFile(listName);
deleteValueListsFile(IPS_LIST_FILE_NAME);
cy.get(VALUE_LISTS_TABLE)
.find(VALUE_LISTS_ROW)
.should(($row) => {
expect($row.text()).not.to.contain(listName);
expect($row.text()).not.to.contain(IPS_LIST_FILE_NAME);
});
});

it('deletes a "ip_range" from an uploaded file', () => {
const listName = 'cidr_list.txt';
importValueList(listName, 'ip_range', ['192.168.100.0']);
importValueList(CIDRS_LIST_FILE_NAME, 'ip_range', ['192.168.100.0']);
openValueListsModal();
deleteValueListsFile(listName);
deleteValueListsFile(CIDRS_LIST_FILE_NAME);
cy.get(VALUE_LISTS_TABLE)
.find(VALUE_LISTS_ROW)
.should(($row) => {
expect($row.text()).not.to.contain(listName);
expect($row.text()).not.to.contain(CIDRS_LIST_FILE_NAME);
});
});
});

describe('export list types', () => {
it('exports a "keyword" list from an uploaded file', () => {
const listName = 'value_list.txt';
cy.intercept('POST', `/api/lists/items/_export?list_id=${listName}`).as('exportList');
importValueList('value_list.txt', 'keyword');
cy.intercept('POST', `/api/lists/items/_export?list_id=${TEXT_LIST_FILE_NAME}`).as(
'exportList'
);
importValueList(TEXT_LIST_FILE_NAME, 'keyword');

// Importing value lists includes bulk creation of list items with refresh=wait_for
// While it should wait for data update and return after that it's not always a case with bulk operations.
// Sometimes list items are empty making this test flaky.
// To fix it refresh used list items index (for the default space)
refreshIndex('.items-default');

openValueListsModal();
exportValueList();

cy.wait('@exportList').then(({ response }) => {
cy.fixture(listName).then((list: string) => {
cy.fixture(TEXT_LIST_FILE_NAME).then((list: string) => {
const [lineOne, lineTwo] = list.split('\n');
expect(response?.body).to.contain(lineOne);
expect(response?.body).to.contain(lineTwo);
Expand All @@ -178,13 +181,22 @@ describe('value lists', () => {
});

it('exports a "text" list from an uploaded file', () => {
const listName = 'value_list.txt';
cy.intercept('POST', `/api/lists/items/_export?list_id=${listName}`).as('exportList');
importValueList(listName, 'text');
cy.intercept('POST', `/api/lists/items/_export?list_id=${TEXT_LIST_FILE_NAME}`).as(
'exportList'
);
importValueList(TEXT_LIST_FILE_NAME, 'text');

// Importing value lists includes bulk creation of list items with refresh=wait_for
// While it should wait for data update and return after that it's not always a case with bulk operations.
// Sometimes list items are empty making this test flaky.
// To fix it refresh used list items index (for the default space)
refreshIndex('.items-default');

openValueListsModal();
exportValueList();

cy.wait('@exportList').then(({ response }) => {
cy.fixture(listName).then((list: string) => {
cy.fixture(TEXT_LIST_FILE_NAME).then((list: string) => {
const [lineOne, lineTwo] = list.split('\n');
expect(response?.body).to.contain(lineOne);
expect(response?.body).to.contain(lineTwo);
Expand All @@ -193,13 +205,21 @@ describe('value lists', () => {
});

it('exports a "ip" list from an uploaded file', () => {
const listName = 'ip_list.txt';
cy.intercept('POST', `/api/lists/items/_export?list_id=${listName}`).as('exportList');
importValueList(listName, 'ip');
cy.intercept('POST', `/api/lists/items/_export?list_id=${IPS_LIST_FILE_NAME}`).as(
'exportList'
);
importValueList(IPS_LIST_FILE_NAME, 'ip');

// Importing value lists includes bulk creation of list items with refresh=wait_for
// While it should wait for data update and return after that it's not always a case with bulk operations.
// Sometimes list items are empty making this test flaky.
// To fix it refresh used list items index (for the default space)
refreshIndex('.items-default');

openValueListsModal();
exportValueList();
cy.wait('@exportList').then(({ response }) => {
cy.fixture(listName).then((list: string) => {
cy.fixture(IPS_LIST_FILE_NAME).then((list: string) => {
const [lineOne, lineTwo] = list.split('\n');
expect(response?.body).to.contain(lineOne);
expect(response?.body).to.contain(lineTwo);
Expand All @@ -208,13 +228,21 @@ describe('value lists', () => {
});

it('exports a "ip_range" list from an uploaded file', () => {
const listName = 'cidr_list.txt';
cy.intercept('POST', `/api/lists/items/_export?list_id=${listName}`).as('exportList');
importValueList(listName, 'ip_range', ['192.168.100.0']);
cy.intercept('POST', `/api/lists/items/_export?list_id=${CIDRS_LIST_FILE_NAME}`).as(
'exportList'
);
importValueList(CIDRS_LIST_FILE_NAME, 'ip_range', ['192.168.100.0']);

// Importing value lists includes bulk creation of list items with refresh=wait_for
// While it should wait for data update and return after that it's not always a case with bulk operations.
// Sometimes list items are empty making this test flaky.
// To fix it refresh used list items index (for the default space)
refreshIndex('.items-default');

openValueListsModal();
exportValueList();
cy.wait('@exportList').then(({ response }) => {
cy.fixture(listName).then((list: string) => {
cy.fixture(CIDRS_LIST_FILE_NAME).then((list: string) => {
const [lineOne] = list.split('\n');
expect(response?.body).to.contain(lineOne);
});
Expand Down
35 changes: 7 additions & 28 deletions x-pack/test/security_solution_cypress/cypress/tasks/lists.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import {
VALUE_LIST_CLOSE_BUTTON,
VALUE_LIST_DELETE_BUTTON,
VALUE_LIST_EXPORT_BUTTON,
VALUE_LIST_FILES,
VALUE_LIST_FILE_PICKER,
VALUE_LIST_FILE_UPLOAD_BUTTON,
VALUE_LIST_TYPE_SELECTOR,
Expand Down Expand Up @@ -73,9 +72,14 @@ export const exportValueList = (): Cypress.Chainable<JQuery<HTMLElement>> => {

/**
* Given an array of value lists this will delete them all using Cypress Request and the lists REST API
*
* If a list doesn't exist it ignores the error.
*
* Ref: https://www.elastic.co/guide/en/security/current/lists-api-delete-container.html
*/
const deleteValueLists = (lists: string[]): Array<Cypress.Chainable<Cypress.Response<unknown>>> => {
export const deleteValueLists = (
lists: string[]
): Array<Cypress.Chainable<Cypress.Response<unknown>>> => {
return lists.map((list) => deleteValueList(list));
};

Expand All @@ -88,6 +92,7 @@ const deleteValueList = (list: string): Cypress.Chainable<Cypress.Response<unkno
method: 'DELETE',
url: `api/lists?id=${list}`,
headers: { 'kbn-xsrf': 'delete-lists', 'x-elastic-internal-origin': 'security-solution' },
failOnStatusCode: false,
});
};

Expand Down Expand Up @@ -149,29 +154,3 @@ export const importValueList = (
) => {
return cy.fixture<string>(file).then((data) => uploadListItemData(file, type, data));
};

/**
* If you are on the value lists from the UI, this will loop over all the HTML elements
* that have action-delete-value-list-${list_name} and delete all of those value lists
* using Cypress Request and the lists REST API.
* If the UI does not contain any value based lists this will not fail. If the UI does
* contain value based lists but the backend does not return a success on DELETE then this
* will cause errors.
* Ref: https://www.elastic.co/guide/en/security/current/lists-api-delete-container.html
*/
export const deleteAllValueListsFromUI = (): Array<
Cypress.Chainable<Cypress.Response<unknown>>
> => {
const lists = Cypress.$(VALUE_LIST_FILES)
.toArray()
.reduce<string[]>((accum, $el) => {
const attribute = $el.getAttribute('data-test-subj');
if (attribute != null) {
const list = attribute.substr('data-test-subj-value-list'.length);
return [...accum, list];
} else {
return accum;
}
}, []);
return deleteValueLists(lists);
};