Skip to content

Commit

Permalink
add security laryer for index patterns privileges
Browse files Browse the repository at this point in the history
  • Loading branch information
@semd
semd committed Jan 19, 2024
1 parent e1d7c9b commit 4517e84
Show file tree
Hide file tree
Showing 8 changed files with 232 additions and 18 deletions.
9 changes: 6 additions & 3 deletions packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ describe('createOrUpdateComponentTemplate', () => {
it(`should update index template field limit and retry if putTemplate throws error with field limit error`, async () => {
clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(
new EsErrors.ResponseError({
body: 'illegal_argument_exception: Limit of total fields [1900] has been exceeded',
body: 'illegal_argument_exception: Limit of total fields [1800] has been exceeded',
} as DiagnosticResult)
);
const existingIndexTemplate = {
Expand Down Expand Up @@ -144,6 +144,7 @@ describe('createOrUpdateComponentTemplate', () => {
totalFieldsLimit: 2500,
});

expect(logger.info).toHaveBeenCalledWith('Updating total_fields.limit from 1800 to 2500');
expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2);
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1);
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({
Expand All @@ -164,7 +165,7 @@ describe('createOrUpdateComponentTemplate', () => {
it(`should update index template field limit and retry if putTemplate throws error with field limit error when there are malformed index templates`, async () => {
clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(
new EsErrors.ResponseError({
body: 'illegal_argument_exception: Limit of total fields [1900] has been exceeded',
body: 'illegal_argument_exception: Limit of total fields [1800] has been exceeded',
} as DiagnosticResult)
);
const existingIndexTemplate = {
Expand Down Expand Up @@ -220,6 +221,7 @@ describe('createOrUpdateComponentTemplate', () => {
totalFieldsLimit: 2500,
});

expect(logger.info).toHaveBeenCalledWith('Updating total_fields.limit from 1800 to 2500');
expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2);
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1);
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({
Expand All @@ -240,7 +242,7 @@ describe('createOrUpdateComponentTemplate', () => {
it(`should retry getIndexTemplate and putIndexTemplate on transient ES errors`, async () => {
clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(
new EsErrors.ResponseError({
body: 'illegal_argument_exception: Limit of total fields [1900] has been exceeded',
body: 'illegal_argument_exception: Limit of total fields [1800] has been exceeded',
} as DiagnosticResult)
);
const existingIndexTemplate = {
Expand Down Expand Up @@ -281,6 +283,7 @@ describe('createOrUpdateComponentTemplate', () => {
totalFieldsLimit: 2500,
});

expect(logger.info).toHaveBeenCalledWith('Updating total_fields.limit from 1800 to 2500');
expect(clusterClient.indices.getIndexTemplate).toHaveBeenCalledTimes(3);
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3);
});
Expand Down
8 changes: 7 additions & 1 deletion packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,20 @@ const createOrUpdateComponentTemplateHelper = async (
try {
await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { logger });
} catch (error) {
if (error.message.match(/Limit of total fields \[\d+\] has been exceeded/) != null) {
const limitErrorMatch = error.message.match(
/Limit of total fields \[(\d+)\] has been exceeded/
);
if (limitErrorMatch != null) {
// This error message occurs when there is an index template using this component template
// that contains a field limit setting that using this component template exceeds
// Specifically, this can happen for the ECS component template when we add new fields
// to adhere to the ECS spec. Individual index templates specify field limits so if the
// number of new ECS fields pushes the composed mapping above the limit, this error will
// occur. We have to update the field limit inside the index template now otherwise we
// can never update the component template

logger.info(`Updating total_fields.limit from ${limitErrorMatch[1]} to ${totalFieldsLimit}`);

await putIndexTemplateTotalFieldsLimitUsingComponentTemplate(
esClient,
template.name,
Expand Down
2 changes: 1 addition & 1 deletion packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import type {
MappingTypeMapping,
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import { isEmpty } from 'lodash';
import { isEmpty } from 'lodash/fp';
import { retryTransientEsErrors } from './retry_transient_es_errors';

interface CreateOrUpdateIndexTemplateOpts {
Expand Down
10 changes: 6 additions & 4 deletions ...curity-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import { useCallback, useEffect, useMemo, useState } from 'react';
import { EcsVersion } from '@kbn/ecs';

import { isEmpty } from 'lodash';
import { isEmpty } from 'lodash/fp';
import {
getTotalDocsCount,
getTotalIncompatible,
Expand Down Expand Up @@ -65,9 +65,12 @@ const useStoredPatternRollups = (patterns: string[]) => {
const [storedRollups, setStoredRollups] = useState<Record<string, PatternRollup>>({});

useEffect(() => {
if (isEmpty(patterns)) {
return;
}

let ignore = false;
const abortController = new AbortController();

const fetchStoredRollups = async () => {
const results = await getResults({ httpFetch, abortController, patterns, toasts });
if (results?.length && !ignore) {
Expand All @@ -79,8 +82,7 @@ const useStoredPatternRollups = (patterns: string[]) => {
return () => {
ignore = true;
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [patterns]);
}, [httpFetch, patterns, toasts]);

return storedRollups;
};
Expand Down
100 changes: 98 additions & 2 deletions x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,13 @@ import { serverMock } from '../../__mocks__/server';
import { requestMock } from '../../__mocks__/request';
import { requestContextMock } from '../../__mocks__/request_context';
import type { LatestAggResponseBucket } from './get_results';
import { getResultsRoute } from './get_results';
import { getResultsRoute, getQuery } from './get_results';
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
import { resultBody, resultDocument } from './results.mock';
import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types';
import type {
SearchResponse,
SecurityHasPrivilegesResponse,
} from '@elastic/elasticsearch/lib/api/types';
import type { ResultDocument } from '../../schemas/result';

const searchResponse = {
Expand Down Expand Up @@ -51,6 +54,10 @@ describe('getResultsRoute route', () => {

({ context } = requestContextMock.createTools());

context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({
index: { 'logs-*': { all: true }, 'alerts-*': { all: true } },
} as unknown as SecurityHasPrivilegesResponse);

getResultsRoute(server.router, logger);
});

Expand Down Expand Up @@ -88,6 +95,95 @@ describe('getResultsRoute route', () => {
});
});

describe('request pattern authorization', () => {
let server: ReturnType<typeof serverMock.create>;
let { context } = requestContextMock.createTools();
let logger: MockedLogger;

const req = requestMock.create({
method: 'get',
path: RESULTS_ROUTE_PATH,
query: { patterns: 'logs-*,alerts-*' },
});

beforeEach(() => {
jest.clearAllMocks();

server = serverMock.create();
logger = loggerMock.create();

({ context } = requestContextMock.createTools());

context.core.elasticsearch.client.asInternalUser.search.mockResolvedValue(searchResponse);

context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({
index: { 'logs-*': { all: true }, 'alerts-*': { all: true } },
} as unknown as SecurityHasPrivilegesResponse);

getResultsRoute(server.router, logger);
});

it('should authorize pattern', async () => {
const mockHasPrivileges =
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
mockHasPrivileges.mockResolvedValueOnce({
index: { 'logs-*': { all: true }, 'alerts-*': { all: true } },
} as unknown as SecurityHasPrivilegesResponse);

const response = await server.inject(req, requestContextMock.convertContext(context));
expect(mockHasPrivileges).toHaveBeenCalledWith({
index: [
{ names: ['logs-*', 'alerts-*'], privileges: ['all', 'read', 'view_index_metadata'] },
],
});
expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalled();

expect(response.status).toEqual(200);
expect(response.body).toEqual([{ '@timestamp': expect.any(Number), ...resultBody }]);
});

it('should search authorized patterns only', async () => {
const mockHasPrivileges =
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
mockHasPrivileges.mockResolvedValueOnce({
index: { 'logs-*': { all: false }, 'alerts-*': { all: true } },
} as unknown as SecurityHasPrivilegesResponse);

const response = await server.inject(req, requestContextMock.convertContext(context));
expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalledWith({
index: expect.any(String),
...getQuery(['alerts-*']),
});

expect(response.status).toEqual(200);
});

it('should not search unauthorized patterns', async () => {
const mockHasPrivileges =
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
mockHasPrivileges.mockResolvedValueOnce({
index: { 'logs-*': { all: false }, 'alerts-*': { all: false } },
} as unknown as SecurityHasPrivilegesResponse);

const response = await server.inject(req, requestContextMock.convertContext(context));
expect(context.core.elasticsearch.client.asInternalUser.search).not.toHaveBeenCalled();

expect(response.status).toEqual(200);
expect(response.body).toEqual([]);
});

it('handles pattern authorization error', async () => {
const errorMessage = 'Error!';
const mockHasPrivileges =
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
mockHasPrivileges.mockRejectedValueOnce({ message: errorMessage });

const response = await server.inject(req, requestContextMock.convertContext(context));
expect(response.status).toEqual(500);
expect(response.body).toEqual({ message: errorMessage, status_code: 500 });
});
});

describe('request validation', () => {
let server: ReturnType<typeof serverMock.create>;
let logger: MockedLogger;
Expand Down
27 changes: 22 additions & 5 deletions x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ import type { DataQualityDashboardRequestHandlerContext } from '../../types';
import { createResultFromDocument } from './parser';
import { API_RESULTS_INDEX_NOT_AVAILABLE } from './translations';

const getQuery = (patterns: string) => ({
export const getQuery = (patterns: string[]) => ({
size: 0,
query: {
bool: { filter: [{ terms: { 'rollup.pattern': patterns.split(',') } }] },
bool: { filter: [{ terms: { 'rollup.pattern': patterns } }] },
},
aggs: {
latest: {
Expand All @@ -43,6 +43,7 @@ export const getResultsRoute = (
.get({
path: RESULTS_ROUTE_PATH,
access: 'internal',
options: { tags: ['access:securitySolution'] },
})
.addVersion(
{
Expand All @@ -65,10 +66,26 @@ export const getResultsRoute = (
}

try {
const query = { index, ...getQuery(request.query.patterns) };
const esClient = services.core.elasticsearch.client.asInternalUser;
// Confirm user has authorization for the requested patterns
const { patterns } = request.query;
const userEsClient = services.core.elasticsearch.client.asCurrentUser;
const privileges = await userEsClient.security.hasPrivileges({
index: [
{ names: patterns.split(','), privileges: ['all', 'read', 'view_index_metadata'] },
],
});
const authorizedPatterns = Object.keys(privileges.index).filter((pattern) =>
Object.values(privileges.index[pattern]).some((v) => v === true)
);
if (authorizedPatterns.length === 0) {
return response.ok({ body: [] });
}

// Get the latest result of each pattern
const query = { index, ...getQuery(authorizedPatterns) };
const internalEsClient = services.core.elasticsearch.client.asInternalUser;

const { aggregations } = await esClient.search<
const { aggregations } = await internalEsClient.search<
ResultDocument,
Record<string, { buckets: LatestAggResponseBucket[] }>
>(query);
Expand Down
82 changes: 80 additions & 2 deletions x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ import { requestMock } from '../../__mocks__/request';
import { requestContextMock } from '../../__mocks__/request_context';
import { postResultsRoute } from './post_results';
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
import type { WriteResponseBase } from '@elastic/elasticsearch/lib/api/types';
import type {
SecurityHasPrivilegesResponse,
WriteResponseBase,
} from '@elastic/elasticsearch/lib/api/types';
import { resultBody, resultDocument } from './results.mock';

describe('postResultsRoute route', () => {
Expand All @@ -29,6 +32,10 @@ describe('postResultsRoute route', () => {

({ context } = requestContextMock.createTools());

context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({
has_all_requested: true,
} as unknown as SecurityHasPrivilegesResponse);

postResultsRoute(server.router, logger);
});

Expand Down Expand Up @@ -58,7 +65,7 @@ describe('postResultsRoute route', () => {
expect(logger.error).toHaveBeenCalledWith(expect.stringContaining(errorMessage));
});

it('handles error', async () => {
it('handles index error', async () => {
const errorMessage = 'Error!';
const mockIndex = context.core.elasticsearch.client.asInternalUser.index;
mockIndex.mockRejectedValueOnce({ message: errorMessage });
Expand All @@ -69,6 +76,77 @@ describe('postResultsRoute route', () => {
});
});

describe('request pattern authorization', () => {
let server: ReturnType<typeof serverMock.create>;
let { context } = requestContextMock.createTools();
let logger: MockedLogger;

const req = requestMock.create({ method: 'post', path: RESULTS_ROUTE_PATH, body: resultBody });

beforeEach(() => {
jest.clearAllMocks();

server = serverMock.create();
logger = loggerMock.create();

({ context } = requestContextMock.createTools());

context.core.elasticsearch.client.asInternalUser.index.mockResolvedValueOnce({
result: 'created',
} as WriteResponseBase);

postResultsRoute(server.router, logger);
});

it('should authorize pattern', async () => {
const mockHasPrivileges =
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
mockHasPrivileges.mockResolvedValueOnce({
has_all_requested: true,
} as unknown as SecurityHasPrivilegesResponse);

const response = await server.inject(req, requestContextMock.convertContext(context));
expect(mockHasPrivileges).toHaveBeenCalledWith({
index: [
{ names: [resultBody.rollup.pattern], privileges: ['all', 'read', 'view_index_metadata'] },
],
});
expect(context.core.elasticsearch.client.asInternalUser.index).toHaveBeenCalled();
expect(response.status).toEqual(200);
expect(response.body).toEqual({ result: 'created' });
});

it('should not index unauthorized pattern', async () => {
const mockHasPrivileges =
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
mockHasPrivileges.mockResolvedValueOnce({
has_all_requested: false,
} as unknown as SecurityHasPrivilegesResponse);

const response = await server.inject(req, requestContextMock.convertContext(context));
expect(mockHasPrivileges).toHaveBeenCalledWith({
index: [
{ names: [resultBody.rollup.pattern], privileges: ['all', 'read', 'view_index_metadata'] },
],
});
expect(context.core.elasticsearch.client.asInternalUser.index).not.toHaveBeenCalled();

expect(response.status).toEqual(200);
expect(response.body).toEqual({ result: 'noop' });
});

it('handles pattern authorization error', async () => {
const errorMessage = 'Error!';
const mockHasPrivileges =
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
mockHasPrivileges.mockRejectedValueOnce({ message: errorMessage });

const response = await server.inject(req, requestContextMock.convertContext(context));
expect(response.status).toEqual(500);
expect(response.body).toEqual({ message: errorMessage, status_code: 500 });
});
});

describe('request validation', () => {
let server: ReturnType<typeof serverMock.create>;
let logger: MockedLogger;
Expand Down
Loading

0 comments on commit 4517e84

Please sign in to comment.