diff --git a/.github/workflows/cypress-workflow.yml b/.github/workflows/cypress-workflow.yml index e34683360..ea311bd20 100644 --- a/.github/workflows/cypress-workflow.yml +++ b/.github/workflows/cypress-workflow.yml @@ -13,7 +13,13 @@ env: jobs: tests: name: Run Cypress E2E tests - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest] + include: + - os: ubuntu-latest + cypress_cache_folder: ~/.cache/Cypress + runs-on: ${{ matrix.os }} env: # prevents extra Cypress installation progress messages CI: 1 @@ -25,55 +31,87 @@ jobs: with: # TODO: Parse this from security analytics plugin (https://github.com/opensearch-project/security-analytics/issues/170) java-version: 11 + - name: Checkout security analytics uses: actions/checkout@v2 with: path: security-analytics repository: opensearch-project/security-analytics ref: ${{ env.SECURITY_ANALYTICS_BRANCH }} + - name: Run opensearch with plugin run: | cd security-analytics ./gradlew run -Dopensearch.version=${{ env.OPENSEARCH_VERSION }} & sleep 300 - # timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done' - - name: Checkout Security Analytics Dashboards plugin - uses: actions/checkout@v2 - with: - path: security-analytics-dashboards-plugin + shell: bash + - name: Checkout OpenSearch-Dashboards uses: actions/checkout@v2 with: repository: opensearch-project/OpenSearch-Dashboards path: OpenSearch-Dashboards ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }} + + - name: Checkout Security Analytics Dashboards plugin + uses: actions/checkout@v2 + with: + path: OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin + - name: Get node and yarn versions id: versions run: | echo "::set-output name=node_version::$(node -p "(require('./OpenSearch-Dashboards/package.json').engines.node).match(/[.0-9]+/)[0]")" echo "::set-output name=yarn_version::$(node -p "(require('./OpenSearch-Dashboards/package.json').engines.yarn).match(/[.0-9]+/)[0]")" + - name: Setup node uses: actions/setup-node@v1 with: node-version: ${{ steps.versions.outputs.node_version }} registry-url: 'https://registry.npmjs.org' + - name: Install correct yarn version for OpenSearch-Dashboards run: | npm uninstall -g yarn echo "Installing yarn ${{ steps.versions_step.outputs.yarn_version }}" npm i -g yarn@${{ steps.versions.outputs.yarn_version }} + - name: Bootstrap plugin/OpenSearch-Dashboards run: | - mkdir -p OpenSearch-Dashboards/plugins - mv security-analytics-dashboards-plugin OpenSearch-Dashboards/plugins cd OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin yarn osd bootstrap + - name: Run OpenSearch-Dashboards server run: | cd OpenSearch-Dashboards yarn start --no-base-path --no-watch & - sleep 300 - # timeout 300 bash -c 'while [[ "$(curl -s localhost:5601/api/status | jq -r '.status.overall.state')" != "green" ]]; do sleep 5; done' + shell: bash + + - name: Sleep until OSD server starts + run: sleep 300 + shell: bash + + - name: Install Cypress + run: | + cd OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin + # This will install Cypress in case the binary is missing which can happen on Windows and Mac + # If the binary exists, this will exit quickly so it should not be an expensive operation + npx cypress install + shell: bash + + - name: Get Cypress version + id: cypress_version + run: | + cd OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin + echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')" + + - name: Cache Cypress + id: cache-cypress + uses: actions/cache@v2 + with: + path: ${{ matrix.cypress_cache_folder }} + key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }} + # for now just chrome, use matrix to do all browsers later - name: Cypress tests uses: cypress-io/github-action@v2 @@ -82,15 +120,19 @@ jobs: command: yarn run cypress run wait-on: 'http://localhost:5601' browser: chrome + env: + CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }} + # Screenshots are only captured on failure, will change this once we do visual regression tests - uses: actions/upload-artifact@v1 if: failure() with: - name: cypress-screenshots + name: cypress-screenshots-${{ matrix.os }} path: OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin/cypress/screenshots + # Test run video was always captured, so this action uses "always()" condition - uses: actions/upload-artifact@v1 if: always() with: - name: cypress-videos + name: cypress-videos-${{ matrix.os }} path: OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin/cypress/videos diff --git a/cypress/integration/1_detectors.spec.js b/cypress/integration/1_detectors.spec.js index 75573f160..cd2f5a909 100644 --- a/cypress/integration/1_detectors.spec.js +++ b/cypress/integration/1_detectors.spec.js @@ -69,7 +69,7 @@ describe('Detectors', () => { cy.get('button').contains('Next').click({ force: true }); // Check that correct page now showing - cy.contains('Required field mappings'); + cy.contains('Configure field mapping'); // Select appropriate names to map fields to for (let field_name in sample_field_mappings.properties) { diff --git a/public/pages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx b/public/pages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx index 4340d12e8..8d5df4ec8 100644 --- a/public/pages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx +++ b/public/pages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx @@ -5,12 +5,20 @@ import React, { Component } from 'react'; import { RouteComponentProps } from 'react-router-dom'; -import { EuiSpacer, EuiTitle, EuiText } from '@elastic/eui'; +import { + EuiSpacer, + EuiTitle, + EuiText, + EuiCallOut, + EuiAccordion, + EuiHorizontalRule, + EuiPanel, +} from '@elastic/eui'; import FieldMappingsTable from '../components/RequiredFieldMapping'; import { createDetectorSteps } from '../../../utils/constants'; import { ContentPanel } from '../../../../../components/ContentPanel'; import { Detector, FieldMapping } from '../../../../../../models/interfaces'; -import { EMPTY_FIELD_MAPPINGS } from '../utils/constants'; +import { EMPTY_FIELD_MAPPINGS_VIEW } from '../utils/constants'; import { DetectorCreationStep } from '../../../models/types'; import { GetFieldMappingViewResponse } from '../../../../../../server/models/interfaces'; import FieldMappingService from '../../../../../services/FieldMappingService'; @@ -49,7 +57,7 @@ export default class ConfigureFieldMapping extends Component< }); this.state = { loading: props.loading || false, - mappingsData: EMPTY_FIELD_MAPPINGS, + mappingsData: EMPTY_FIELD_MAPPINGS_VIEW, createdMappings, invalidMappingFieldNames: [], }; @@ -70,17 +78,21 @@ export default class ConfigureFieldMapping extends Component< Object.keys(mappingsView.response.properties).forEach((ruleFieldName) => { existingMappings[ruleFieldName] = mappingsView.response.properties[ruleFieldName].path; }); - this.setState({ createdMappings: existingMappings, mappingsData: mappingsView.response }); + this.setState({ + createdMappings: existingMappings, + mappingsData: { + ...mappingsView.response, + unmapped_field_aliases: [ + 'timestamp', + ...(mappingsView.response.unmapped_field_aliases || []), + ], + }, + }); this.updateMappingSharedState(existingMappings); } this.setState({ loading: false }); }; - validateMappings(mappings: ruleFieldToIndexFieldMap): boolean { - // TODO: Implement validation - return true; //allFieldsMapped; // && allAliasesUnique; - } - /** * Returns the fieldName(s) that have duplicate alias assigned to them */ @@ -110,8 +122,7 @@ export default class ConfigureFieldMapping extends Component< invalidMappingFieldNames: invalidMappingFieldNames, }); this.updateMappingSharedState(newMappings); - const mappingsValid = this.validateMappings(newMappings); - this.props.updateDataValidState(DetectorCreationStep.CONFIGURE_FIELD_MAPPING, mappingsValid); + this.props.updateDataValidState(DetectorCreationStep.CONFIGURE_FIELD_MAPPING, true); }; updateMappingSharedState = (createdMappings: ruleFieldToIndexFieldMap) => { @@ -126,42 +137,52 @@ export default class ConfigureFieldMapping extends Component< }; render() { - const { isEdit } = this.props; const { loading, mappingsData, createdMappings, invalidMappingFieldNames } = this.state; const existingMappings: ruleFieldToIndexFieldMap = { ...createdMappings, }; - const ruleFields = [...(mappingsData.unmapped_field_aliases || [])]; - const indexFields = [...(mappingsData.unmapped_index_fields || [])]; + // read only data + const mappedRuleFields: string[] = []; + const mappedLogFields: string[] = []; Object.keys(mappingsData.properties).forEach((ruleFieldName) => { - existingMappings[ruleFieldName] = mappingsData.properties[ruleFieldName].path; - ruleFields.unshift(ruleFieldName); - indexFields.unshift(mappingsData.properties[ruleFieldName].path); + mappedRuleFields.unshift(ruleFieldName); + mappedLogFields.unshift(mappingsData.properties[ruleFieldName].path); }); + // edit data + const ruleFields = [...(mappingsData.unmapped_field_aliases || [])]; + const indexFields = [...(mappingsData.unmapped_index_fields || [])]; + return (
- {!isEdit && ( - <> - -

{createDetectorSteps[DetectorCreationStep.CONFIGURE_FIELD_MAPPING].title}

- + +

{createDetectorSteps[DetectorCreationStep.CONFIGURE_FIELD_MAPPING].title}

+ - - To perform threat detection, known field names from your log data source are - automatically mapped to rule field names. Additional fields that may require manual - mapping will be shown below. - + + To perform threat detection, known field names from your log data source are automatically + mapped to rule field names. Additional fields that may require manual mapping will be + shown below. + - - - )} + - {ruleFields.length > 0 && ( + {ruleFields.length > 0 ? ( <> - + +

+ To generate accurate findings, we recommend mapping the following security rules + fields with the log field from your data source. +

+ + + + {...this.props} loading={loading} ruleFields={ruleFields} indexFields={indexFields} @@ -171,12 +192,48 @@ export default class ConfigureFieldMapping extends Component< invalidMappingFieldNames, onMappingCreation: this.onMappingCreation, }} - {...this.props} /> + ) : ( + <> + +

+ Your data source(s) have been mapped with all security rule fields. No action is + needed. +

+ + + )} + + + + +

{`View mapped fields (${mappedRuleFields.length})`}

+ +
+ } + buttonProps={{ style: { paddingLeft: '10px', paddingRight: '10px' } }} + id={'mappedFieldsAccordion'} + initialIsOpen={false} + > + + + {...this.props} + loading={loading} + ruleFields={mappedRuleFields} + indexFields={mappedLogFields} + mappingProps={{ + type: MappingViewType.Readonly, + }} + /> + + + ); } diff --git a/public/pages/CreateDetector/components/ConfigureFieldMapping/utils/constants.ts b/public/pages/CreateDetector/components/ConfigureFieldMapping/utils/constants.ts index e1245b54e..cd7e111fc 100644 --- a/public/pages/CreateDetector/components/ConfigureFieldMapping/utils/constants.ts +++ b/public/pages/CreateDetector/components/ConfigureFieldMapping/utils/constants.ts @@ -3,15 +3,22 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { GetFieldMappingViewResponse } from '../../../../../../server/models/interfaces'; +import { + FieldMappingPropertyMap, + GetFieldMappingViewResponse, +} from '../../../../../../server/models/interfaces'; export const STATUS_ICON_PROPS = { unmapped: { type: 'alert', color: 'danger' }, mapped: { type: 'checkInCircleFilled', color: 'success' }, }; -export const EMPTY_FIELD_MAPPINGS: GetFieldMappingViewResponse = { +export const EMPTY_FIELD_MAPPINGS_VIEW: GetFieldMappingViewResponse = { properties: {}, unmapped_field_aliases: [], unmapped_index_fields: [], }; + +export const EMPTY_FIELD_MAPPINGS: FieldMappingPropertyMap = { + properties: {}, +}; diff --git a/public/pages/Detectors/components/FieldMappingsView/FieldMappingsView.tsx b/public/pages/Detectors/components/FieldMappingsView/FieldMappingsView.tsx index 3d8ea5cb4..06c782874 100644 --- a/public/pages/Detectors/components/FieldMappingsView/FieldMappingsView.tsx +++ b/public/pages/Detectors/components/FieldMappingsView/FieldMappingsView.tsx @@ -52,10 +52,10 @@ export const FieldMappingsView: React.FC = ({ async (indexName: string) => { const getMappingRes = await services?.fieldMappingService.getMappings(indexName); if (getMappingRes?.ok) { - const mappings = getMappingRes.response[detector.detector_type.toLowerCase()]; - if (mappings) { + const mappingsData = getMappingRes.response[indexName]; + if (mappingsData) { let items: FieldMappingsTableItem[] = []; - Object.entries(mappings.mappings.properties).forEach((entry) => { + Object.entries(mappingsData.mappings.properties).forEach((entry) => { items.push({ ruleFieldName: entry[0], logFieldName: entry[1].path, diff --git a/public/pages/Detectors/components/UpdateFieldMappings/UpdateFieldMappings.tsx b/public/pages/Detectors/components/UpdateFieldMappings/UpdateFieldMappings.tsx index eb5a6f5b5..1ee5ec8af 100644 --- a/public/pages/Detectors/components/UpdateFieldMappings/UpdateFieldMappings.tsx +++ b/public/pages/Detectors/components/UpdateFieldMappings/UpdateFieldMappings.tsx @@ -6,7 +6,6 @@ import React, { Component } from 'react'; import { RouteComponentProps } from 'react-router-dom'; import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiTitle } from '@elastic/eui'; -import ConfigureFieldMapping from '../../../CreateDetector/components/ConfigureFieldMapping'; import { Detector, FieldMapping } from '../../../../../models/interfaces'; import FieldMappingService from '../../../../services/FieldMappingService'; import { DetectorHit, SearchDetectorsResponse } from '../../../../../server/models/interfaces'; @@ -15,6 +14,7 @@ import { DetectorsService } from '../../../../services'; import { ServerResponse } from '../../../../../server/models/types'; import { NotificationsStart } from 'opensearch-dashboards/public'; import { errorNotificationToast, successNotificationToast } from '../../../../utils/helpers'; +import EditFieldMappings from '../../containers/FieldMappings/EditFieldMapping'; export interface UpdateFieldMappingsProps extends RouteComponentProps { @@ -159,14 +159,12 @@ export default class UpdateFieldMappings extends Component< {!loading && ( - {}} loading={loading} /> )} diff --git a/public/pages/Detectors/containers/FieldMappings/EditFieldMapping.tsx b/public/pages/Detectors/containers/FieldMappings/EditFieldMapping.tsx new file mode 100644 index 000000000..a99c089c5 --- /dev/null +++ b/public/pages/Detectors/containers/FieldMappings/EditFieldMapping.tsx @@ -0,0 +1,157 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { Component } from 'react'; +import { RouteComponentProps } from 'react-router-dom'; +import { EuiSpacer } from '@elastic/eui'; +import FieldMappingsTable from '../../../CreateDetector/components/ConfigureFieldMapping/components/RequiredFieldMapping'; +import { ContentPanel } from '../../../../components/ContentPanel'; +import { Detector, FieldMapping } from '../../../../../models/interfaces'; +import { EMPTY_FIELD_MAPPINGS } from '../../../CreateDetector/components/ConfigureFieldMapping/utils/constants'; +import { FieldMappingPropertyMap } from '../../../../../server/models/interfaces'; +import FieldMappingService from '../../../../services/FieldMappingService'; +import { MappingViewType } from '../../../CreateDetector/components/ConfigureFieldMapping/components/RequiredFieldMapping/FieldMappingsTable'; + +export interface ruleFieldToIndexFieldMap { + [fieldName: string]: string; +} + +interface EditFieldMappingsProps extends RouteComponentProps { + detector: Detector; + filedMappingService: FieldMappingService; + fieldMappings: FieldMapping[]; + loading: boolean; + replaceFieldMappings: (mappings: FieldMapping[]) => void; +} + +interface EditFieldMappingsState { + loading: boolean; + mappingsData: FieldMappingPropertyMap; + createdMappings: ruleFieldToIndexFieldMap; + invalidMappingFieldNames: string[]; +} + +export default class EditFieldMappings extends Component< + EditFieldMappingsProps, + EditFieldMappingsState +> { + constructor(props: EditFieldMappingsProps) { + super(props); + const createdMappings: ruleFieldToIndexFieldMap = {}; + props.fieldMappings.forEach((mapping) => { + createdMappings[mapping.ruleFieldName] = mapping.indexFieldName; + }); + this.state = { + loading: props.loading || false, + mappingsData: EMPTY_FIELD_MAPPINGS, + createdMappings, + invalidMappingFieldNames: [], + }; + } + + componentDidMount = async () => { + this.getAllMappings(); + }; + + getAllMappings = async () => { + this.setState({ loading: true }); + const indexName = this.props.detector.inputs[0].detector_input.indices[0]; + const mappingsView = await this.props.filedMappingService.getMappings(indexName); + if (mappingsView.ok) { + const existingMappings = { ...this.state.createdMappings }; + const properties = mappingsView.response[indexName]?.mappings.properties; + + if (properties) { + Object.keys(properties).forEach((ruleFieldName) => { + existingMappings[ruleFieldName] = properties[ruleFieldName].path; + }); + this.setState({ createdMappings: existingMappings, mappingsData: { properties } }); + this.updateMappingSharedState(existingMappings); + } + } + this.setState({ loading: false }); + }; + + /** + * Returns the fieldName(s) that have duplicate alias assigned to them + */ + getInvalidMappingFieldNames(mappings: ruleFieldToIndexFieldMap): string[] { + const seenAliases = new Set(); + const invalidFields: string[] = []; + + Object.entries(mappings).forEach((entry) => { + if (seenAliases.has(entry[1])) { + invalidFields.push(entry[0]); + } + + seenAliases.add(entry[1]); + }); + + return invalidFields; + } + + onMappingCreation = (ruleFieldName: string, indxFieldName: string): void => { + const newMappings: ruleFieldToIndexFieldMap = { + ...this.state.createdMappings, + [ruleFieldName]: indxFieldName, + }; + const invalidMappingFieldNames = this.getInvalidMappingFieldNames(newMappings); + this.setState({ + createdMappings: newMappings, + invalidMappingFieldNames: invalidMappingFieldNames, + }); + this.updateMappingSharedState(newMappings); + }; + + updateMappingSharedState = (createdMappings: ruleFieldToIndexFieldMap) => { + this.props.replaceFieldMappings( + Object.entries(createdMappings).map((entry) => { + return { + ruleFieldName: entry[0], + indexFieldName: entry[1], + }; + }) + ); + }; + + render() { + const { loading, mappingsData, createdMappings, invalidMappingFieldNames } = this.state; + const existingMappings: ruleFieldToIndexFieldMap = { + ...createdMappings, + }; + const ruleFields: string[] = []; + const indexFields: string[] = []; + + Object.keys(mappingsData.properties).forEach((ruleFieldName) => { + existingMappings[ruleFieldName] = mappingsData.properties[ruleFieldName].path; + ruleFields.unshift(ruleFieldName); + indexFields.unshift(mappingsData.properties[ruleFieldName].path); + }); + + return ( +
+ {ruleFields.length > 0 && ( + <> + + + {...this.props} + loading={loading} + ruleFields={ruleFields} + indexFields={indexFields} + mappingProps={{ + type: MappingViewType.Edit, + existingMappings, + invalidMappingFieldNames, + onMappingCreation: this.onMappingCreation, + }} + /> + + + + )} +
+ ); + } +}