From d18c40cee187c587d175c1ad6a2d90ac3b191458 Mon Sep 17 00:00:00 2001 From: Jason Stoltzfus Date: Tue, 2 Jun 2020 13:59:58 -0400 Subject: [PATCH] E2E testing (#6) * Wired up basics for E2E testing * Added version with App Search * Updated naming * Switched configuration around * Added concept of 'fixtures' * Figured out how to log in as the enterprise_search user * Refactored to use an App Search service * Added some real tests * Added a README * Cleanup * More cleanup * Error handling + README updatre * Removed unnecessary files * Apply suggestions from code review Co-authored-by: Constance * Update x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_table.tsx Co-authored-by: Constance * PR feedback - updated README * Additional lint fixes Co-authored-by: Constance --- .../engine_overview/engine_overview.tsx | 4 +- .../engine_overview/engine_table.tsx | 6 +- .../functional_enterprise_search/README.md | 41 ++++++ .../app_search/engines.ts | 75 +++++++++++ .../with_host_configured/index.ts | 13 ++ .../app_search/setup_guide.ts | 36 ++++++ .../without_host_configured/index.ts | 13 ++ .../base_config.ts | 20 +++ .../page_objects/app_search.ts | 29 +++++ .../page_objects/index.ts | 13 ++ .../services/app_search_client.ts | 121 ++++++++++++++++++ .../services/app_search_service.ts | 83 ++++++++++++ .../services/index.ts | 13 ++ .../with_host_configured.config.ts | 31 +++++ .../without_host_configured.config.ts | 23 ++++ 15 files changed, 518 insertions(+), 3 deletions(-) create mode 100644 x-pack/test/functional_enterprise_search/README.md create mode 100644 x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/app_search/engines.ts create mode 100644 x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/index.ts create mode 100644 x-pack/test/functional_enterprise_search/apps/enterprise_search/without_host_configured/app_search/setup_guide.ts create mode 100644 x-pack/test/functional_enterprise_search/apps/enterprise_search/without_host_configured/index.ts create mode 100644 x-pack/test/functional_enterprise_search/base_config.ts create mode 100644 x-pack/test/functional_enterprise_search/page_objects/app_search.ts create mode 100644 x-pack/test/functional_enterprise_search/page_objects/index.ts create mode 100644 x-pack/test/functional_enterprise_search/services/app_search_client.ts create mode 100644 x-pack/test/functional_enterprise_search/services/app_search_service.ts create mode 100644 x-pack/test/functional_enterprise_search/services/index.ts create mode 100644 x-pack/test/functional_enterprise_search/with_host_configured.config.ts create mode 100644 x-pack/test/functional_enterprise_search/without_host_configured.config.ts diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_overview.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_overview.tsx index 0d9790b4395f9..8676f738a412a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_overview.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_overview.tsx @@ -108,7 +108,7 @@ export const EngineOverview: ReactFC<> = () => { - + = () => { - + = ({ name: i18n.translate('xpack.appSearch.enginesOverview.table.column.name', { defaultMessage: 'Name', }), - render: (name) => {name}, + render: (name) => ( + + {name} + + ), width: '30%', truncateText: true, mobileOptions: { diff --git a/x-pack/test/functional_enterprise_search/README.md b/x-pack/test/functional_enterprise_search/README.md new file mode 100644 index 0000000000000..e5d9009fc393b --- /dev/null +++ b/x-pack/test/functional_enterprise_search/README.md @@ -0,0 +1,41 @@ +# Enterprise Search Functional E2E Tests + +## Running these tests + +Follow the [Functional Test Runner instructions](https://www.elastic.co/guide/en/kibana/current/development-functional-tests.html#_running_functional_tests). + +There are two suites available to run, a suite that requires a Kibana instance without an `enterpriseSearch.host` +configured, and one that does. The later also [requires a running Enterprise Search instance](#enterprise-search-requirement), and a Private API key +from that instance set in an Environment variable. + +Ex. + +```sh +# Run specs from the x-pack directory +cd x-pack + +# Run tests that require enterpriseSearch.host variable +APP_SEARCH_API_KEY=[use private key from local App Search instance here] node scripts/functional_tests --config test/functional_enterprise_search/with_host_configured.config.ts + +# Run tests that do not require enterpriseSearch.host variable +APP_SEARCH_API_KEY=[use private key from local App Search instance here] node scripts/functional_tests --config test/functional_enterprise_search/without_host_configured.config.ts +``` + +## Enterprise Search Requirement + +These tests will not currently start an instance of App Search automatically. As such, they are not run as part of CI and are most useful for local regression testing. + +The easiest way to start Enterprise Search for these tests is to check out the `ent-search` project +and use the following script. + +```sh +cd script/stack_scripts +/start-with-license-and-expiration.sh platinum 500000 +``` + +Requirements for Enterprise Search: + +- Running on port 3002 against a separate Elasticsearch cluster. +- Elasticsearch must have a platinum or greater level license (or trial). +- Must have Standard or Native Auth configured with an `enterprise_search` user with password `changeme`. +- There should be NO existing Engines or Meta Engines. diff --git a/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/app_search/engines.ts b/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/app_search/engines.ts new file mode 100644 index 0000000000000..38bdb429b8e09 --- /dev/null +++ b/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/app_search/engines.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { EsArchiver } from 'src/es_archiver'; +import { AppSearchService, IEngine } from '../../../../services/app_search_service'; +import { Browser } from '../../../../../../../test/functional/services/browser'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function enterpriseSearchSetupEnginesTests({ + getService, + getPageObjects, +}: FtrProviderContext) { + const esArchiver = getService('esArchiver') as EsArchiver; + const browser = getService('browser') as Browser; + const retry = getService('retry'); + const appSearch = getService('appSearch') as AppSearchService; + + const PageObjects = getPageObjects(['appSearch', 'security']); + + describe('Engines Overview', function () { + let engine1: IEngine; + let engine2: IEngine; + let metaEngine: IEngine; + + before(async () => { + await esArchiver.load('empty_kibana'); + engine1 = await appSearch.createEngine(); + engine2 = await appSearch.createEngine(); + metaEngine = await appSearch.createMetaEngine([engine1.name, engine2.name]); + }); + + after(async () => { + await esArchiver.unload('empty_kibana'); + appSearch.destroyEngine(engine1.name); + appSearch.destroyEngine(engine2.name); + appSearch.destroyEngine(metaEngine.name); + }); + + describe('when an enterpriseSearch.host is configured', () => { + it('navigating to the enterprise_search plugin will redirect a user to the App Search Engines Overview page', async () => { + await PageObjects.security.forceLogout(); + const { user, password } = appSearch.getEnterpriseSearchUser(); + await PageObjects.security.login(user, password, { + expectSpaceSelector: false, + }); + + await PageObjects.appSearch.navigateToPage(); + await retry.try(async function () { + const currentUrl = await browser.getCurrentUrl(); + expect(currentUrl).to.contain('/app_search'); + }); + }); + + it('lists engines', async () => { + const engineLinks = await PageObjects.appSearch.getEngineLinks(); + const engineLinksText = await Promise.all(engineLinks.map((l) => l.getVisibleText())); + + expect(engineLinksText.includes(engine1.name)).to.equal(true); + expect(engineLinksText.includes(engine2.name)).to.equal(true); + }); + + it('lists meta engines', async () => { + const metaEngineLinks = await PageObjects.appSearch.getMetaEngineLinks(); + const metaEngineLinksText = await Promise.all( + metaEngineLinks.map((l) => l.getVisibleText()) + ); + expect(metaEngineLinksText.includes(metaEngine.name)).to.equal(true); + }); + }); + }); +} diff --git a/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/index.ts b/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/index.ts new file mode 100644 index 0000000000000..d239d538290fa --- /dev/null +++ b/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Enterprise Search', function () { + loadTestFile(require.resolve('./app_search/engines')); + }); +} diff --git a/x-pack/test/functional_enterprise_search/apps/enterprise_search/without_host_configured/app_search/setup_guide.ts b/x-pack/test/functional_enterprise_search/apps/enterprise_search/without_host_configured/app_search/setup_guide.ts new file mode 100644 index 0000000000000..c328d0b202647 --- /dev/null +++ b/x-pack/test/functional_enterprise_search/apps/enterprise_search/without_host_configured/app_search/setup_guide.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function enterpriseSearchSetupGuideTests({ + getService, + getPageObjects, +}: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + const retry = getService('retry'); + + const PageObjects = getPageObjects(['appSearch']); + + describe('Setup Guide', function () { + before(async () => await esArchiver.load('empty_kibana')); + after(async () => { + await esArchiver.unload('empty_kibana'); + }); + + describe('when no enterpriseSearch.host is configured', () => { + it('navigating to the enterprise_search plugin will redirect a user to the setup guide', async () => { + await PageObjects.appSearch.navigateToPage(); + await retry.try(async function () { + const currentUrl = await browser.getCurrentUrl(); + expect(currentUrl).to.contain('/app_search/setup_guide'); + }); + }); + }); + }); +} diff --git a/x-pack/test/functional_enterprise_search/apps/enterprise_search/without_host_configured/index.ts b/x-pack/test/functional_enterprise_search/apps/enterprise_search/without_host_configured/index.ts new file mode 100644 index 0000000000000..8408af99b117f --- /dev/null +++ b/x-pack/test/functional_enterprise_search/apps/enterprise_search/without_host_configured/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Enterprise Search', function () { + loadTestFile(require.resolve('./app_search/setup_guide')); + }); +} diff --git a/x-pack/test/functional_enterprise_search/base_config.ts b/x-pack/test/functional_enterprise_search/base_config.ts new file mode 100644 index 0000000000000..f737b6cd4b5f4 --- /dev/null +++ b/x-pack/test/functional_enterprise_search/base_config.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; +import { pageObjects } from './page_objects'; +import { services } from './services'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const xPackFunctionalConfig = await readConfigFile(require.resolve('../functional/config')); + + return { + // default to the xpack functional config + ...xPackFunctionalConfig.getAll(), + services, + pageObjects, + }; +} diff --git a/x-pack/test/functional_enterprise_search/page_objects/app_search.ts b/x-pack/test/functional_enterprise_search/page_objects/app_search.ts new file mode 100644 index 0000000000000..a8b40b7774f78 --- /dev/null +++ b/x-pack/test/functional_enterprise_search/page_objects/app_search.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; +import { TestSubjects } from '../../../../../test/functional/services/test_subjects'; + +export function AppSearchPageProvider({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['common']); + const testSubjects = getService('testSubjects') as TestSubjects; + + return { + async navigateToPage() { + return await PageObjects.common.navigateToApp('app_search'); + }, + + async getEngineLinks() { + const engines = await testSubjects.find('appSearchEngines'); + return await testSubjects.findAllDescendant('engineNameLink', engines); + }, + + async getMetaEngineLinks() { + const metaEngines = await testSubjects.find('appSearchMetaEngines'); + return await testSubjects.findAllDescendant('engineNameLink', metaEngines); + }, + }; +} diff --git a/x-pack/test/functional_enterprise_search/page_objects/index.ts b/x-pack/test/functional_enterprise_search/page_objects/index.ts new file mode 100644 index 0000000000000..009fb26482419 --- /dev/null +++ b/x-pack/test/functional_enterprise_search/page_objects/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { pageObjects as basePageObjects } from '../../functional/page_objects'; +import { AppSearchPageProvider } from './app_search'; + +export const pageObjects = { + ...basePageObjects, + appSearch: AppSearchPageProvider, +}; diff --git a/x-pack/test/functional_enterprise_search/services/app_search_client.ts b/x-pack/test/functional_enterprise_search/services/app_search_client.ts new file mode 100644 index 0000000000000..11c383eb779d6 --- /dev/null +++ b/x-pack/test/functional_enterprise_search/services/app_search_client.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import http from 'http'; + +/** + * A simple request client for making API calls to the App Search API + */ +const makeRequest = (method: string, path: string, body?: object): Promise => { + return new Promise(function (resolve, reject) { + const APP_SEARCH_API_KEY = process.env.APP_SEARCH_API_KEY; + + if (!APP_SEARCH_API_KEY) { + throw new Error('Please provide a valid APP_SEARCH_API_KEY. See README for more details.'); + } + + let postData; + + if (body) { + postData = JSON.stringify(body); + } + + const req = http.request( + { + method, + hostname: 'localhost', + port: 3002, + path, + agent: false, // Create a new agent just for this one request + headers: { + Authorization: `Bearer ${APP_SEARCH_API_KEY}`, + 'Content-Type': 'application/json', + ...(!!postData && { 'Content-Length': Buffer.byteLength(postData) }), + }, + }, + (res) => { + const bodyChunks: Uint8Array[] = []; + res.on('data', function (chunk) { + bodyChunks.push(chunk); + }); + + res.on('end', function () { + let responseBody; + try { + responseBody = JSON.parse(Buffer.concat(bodyChunks).toString()); + } catch (e) { + reject(e); + } + + if (res.statusCode > 299) { + reject('Error calling App Search API: ' + JSON.stringify(responseBody)); + } + + resolve(responseBody); + }); + } + ); + + req.on('error', (e) => { + reject(e); + }); + + if (postData) { + req.write(postData); + } + req.end(); + }); +}; + +export interface IEngine { + name: string; +} + +export const createEngine = async (engineName: string): Promise => { + return await makeRequest('POST', '/api/as/v1/engines', { name: engineName }); +}; + +export const destroyEngine = async (engineName: string): Promise => { + return await makeRequest('DELETE', `/api/as/v1/engines/${engineName}`); +}; + +export const createMetaEngine = async ( + engineName: string, + sourceEngines: string[] +): Promise => { + return await makeRequest('POST', '/api/as/v1/engines', { + name: engineName, + type: 'meta', + source_engines: sourceEngines, + }); +}; + +export interface ISearchResponse { + results: object[]; +} + +const search = async (engineName: string): Promise => { + return await makeRequest('POST', `/api/as/v1/engines/${engineName}/search`, { query: '' }); +}; + +// Since the App Search API does not issue document receipts, the only way to tell whether or not documents +// are fully indexed is to poll the search endpoint. +export const waitForIndexedDocs = (engineName: string) => { + return new Promise(async function (resolve) { + let isReady = false; + while (!isReady) { + const response = await search(engineName); + if (response.results && response.results.length > 0) { + isReady = true; + resolve(); + } + } + }); +}; + +export const indexData = async (engineName: string, docs: object[]) => { + return await makeRequest('POST', `/api/as/v1/engines/${engineName}/documents`, docs); +}; diff --git a/x-pack/test/functional_enterprise_search/services/app_search_service.ts b/x-pack/test/functional_enterprise_search/services/app_search_service.ts new file mode 100644 index 0000000000000..c04988a26d5f9 --- /dev/null +++ b/x-pack/test/functional_enterprise_search/services/app_search_service.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +const ENTERPRISE_SEARCH_USER = 'enterprise_search'; +const ENTERPRISE_SEARCH_PASSWORD = 'changeme'; +import { + createEngine, + createMetaEngine, + indexData, + waitForIndexedDocs, + destroyEngine, + IEngine, +} from './app_search_client'; + +export interface IUser { + user: string; + password: string; +} +export { IEngine }; + +export class AppSearchService { + getEnterpriseSearchUser(): IUser { + return { + user: ENTERPRISE_SEARCH_USER, + password: ENTERPRISE_SEARCH_PASSWORD, + }; + } + + createEngine(): Promise { + const engineName = `test-engine-${new Date().getTime()}`; + return createEngine(engineName); + } + + async createEngineWithDocs(): Promise { + const engine = await this.createEngine(); + const docs = [ + { id: 1, name: 'doc1' }, + { id: 2, name: 'doc2' }, + { id: 3, name: 'doc2' }, + ]; + await indexData(engine.name, docs); + await waitForIndexedDocs(engine.name); + return engine; + } + + createMetaEngine(sourceEngines: string[]): Promise { + const engineName = `test-meta-engine-${new Date().getTime()}`; + return createMetaEngine(engineName, sourceEngines); + } + + destroyEngine(engineName: string) { + return destroyEngine(engineName); + } +} + +export async function AppSearchServiceProvider({ getService }: FtrProviderContext) { + const lifecycle = getService('lifecycle'); + const security = getService('security'); + + lifecycle.beforeTests.add(async () => { + const APP_SEARCH_API_KEY = process.env.APP_SEARCH_API_KEY; + + if (!APP_SEARCH_API_KEY) { + throw new Error('Please provide a valid APP_SEARCH_API_KEY. See README for more details.'); + } + + // The App Search plugin passes through the current user name and password + // through on the API call to App Search. Therefore, we need to be signed + // in as the enterprise_search user in order for this plugin to work. + await security.user.create(ENTERPRISE_SEARCH_USER, { + password: ENTERPRISE_SEARCH_PASSWORD, + roles: ['kibana_admin'], + full_name: ENTERPRISE_SEARCH_USER, + }); + }); + + return new AppSearchService(); +} diff --git a/x-pack/test/functional_enterprise_search/services/index.ts b/x-pack/test/functional_enterprise_search/services/index.ts new file mode 100644 index 0000000000000..1715c98677ac6 --- /dev/null +++ b/x-pack/test/functional_enterprise_search/services/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { services as functionalServices } from '../../functional/services'; +import { AppSearchServiceProvider } from './app_search_service'; + +export const services = { + ...functionalServices, + appSearch: AppSearchServiceProvider, +}; diff --git a/x-pack/test/functional_enterprise_search/with_host_configured.config.ts b/x-pack/test/functional_enterprise_search/with_host_configured.config.ts new file mode 100644 index 0000000000000..f425f806f4bcd --- /dev/null +++ b/x-pack/test/functional_enterprise_search/with_host_configured.config.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { resolve } from 'path'; +import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const baseConfig = await readConfigFile(require.resolve('./base_config')); + + return { + // default to the xpack functional config + ...baseConfig.getAll(), + + testFiles: [resolve(__dirname, './apps/enterprise_search/with_host_configured')], + + junit: { + reportName: 'X-Pack Enterprise Search Functional Tests with Host Configured', + }, + + kbnTestServer: { + ...baseConfig.get('kbnTestServer'), + serverArgs: [ + ...baseConfig.get('kbnTestServer.serverArgs'), + '--enterpriseSearch.host=http://localhost:3002', + ], + }, + }; +} diff --git a/x-pack/test/functional_enterprise_search/without_host_configured.config.ts b/x-pack/test/functional_enterprise_search/without_host_configured.config.ts new file mode 100644 index 0000000000000..0f2afd214abed --- /dev/null +++ b/x-pack/test/functional_enterprise_search/without_host_configured.config.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { resolve } from 'path'; +import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const baseConfig = await readConfigFile(require.resolve('./base_config')); + + return { + // default to the xpack functional config + ...baseConfig.getAll(), + + testFiles: [resolve(__dirname, './apps/enterprise_search/without_host_configured')], + + junit: { + reportName: 'X-Pack Enterprise Search Functional Tests without Host Configured', + }, + }; +}