Skip to content

Commit

Permalink
[8.x][Index management] Project level retention support (#193715) (#1…
Browse files Browse the repository at this point in the history
…97414)

# Backport

This will backport the following commits from `main` to `8.x`:
 - #193715

Note: Created the backport manually because of merge conflicts in
`config/serverless.security.yml`

Co-authored-by: Ignacio Rivas <[email protected]>
  • Loading branch information
ElenaStoeva and sabarasaba authored Oct 23, 2024
1 parent 722dd5d commit 47f11dd
Show file tree
Hide file tree
Showing 25 changed files with 422 additions and 57 deletions.
3 changes: 3 additions & 0 deletions config/serverless.security.yml
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,9 @@ xpack.ml.compatibleModuleType: 'security'
# Disable the embedded Dev Console
console.ui.embeddedEnabled: false

# Enable project level rentention checks in DSL form from Index Management UI
xpack.index_management.enableProjectLevelRetentionChecks: true

# Experimental Security Solution features

# This feature is disabled in Serverless until fully performance tested within a Serverless environment
Expand Down
2 changes: 2 additions & 0 deletions config/serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ xpack.index_management.editableIndexSettings: limited
xpack.index_management.enableMappingsSourceFieldSection: false
# Disable toggle for enabling data retention in DSL form from Index Management UI
xpack.index_management.enableTogglingDataRetention: false
# Disable project level rentention checks in DSL form from Index Management UI
xpack.index_management.enableProjectLevelRetentionChecks: false

# Disable Manage Processors UI in Ingest Pipelines
xpack.ingest_pipelines.enableManageProcessors: false
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
'xpack.index_management.ui.enabled (boolean?)',
'xpack.infra.sources.default.fields.message (array?)',
'xpack.index_management.enableTogglingDataRetention (boolean?|never)',
'xpack.index_management.enableProjectLevelRetentionChecks (boolean?|never)',
'xpack.integration_assistant.enableExperimental (array?)',
/**
* Feature flags bellow are conditional based on traditional/serverless offering
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ export type TestSubjects =
| 'configuredByILMWarning'
| 'show-filters-button'
| 'filter-option-h'
| 'filter-option-d'
| 'infiniteRetentionPeriod.input'
| 'saveButton'
| 'dsIsFullyManagedByILM'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { act } from 'react-dom/test-utils';
import { createMemoryHistory } from 'history';
import { notificationServiceMock } from '@kbn/core/public/mocks';

import { breadcrumbService } from '../../../public/application/services/breadcrumbs';
import { MAX_DATA_RETENTION } from '../../../common/constants';
import * as fixtures from '../../../test/fixtures';
import { setupEnvironment } from '../helpers';
import { notificationService } from '../../../public/application/services/notification';

import {
DataStreamsTabTestBed,
setup,
createDataStreamPayload,
createDataStreamBackingIndex,
createNonDataStreamIndex,
} from './data_streams_tab.helpers';

const urlServiceMock = {
locators: {
get: () => ({
getLocation: async () => ({
app: '',
path: '',
state: {},
}),
getUrl: async ({ policyName }: { policyName: string }) => `/test/${policyName}`,
navigate: async () => {},
useUrl: () => '',
}),
},
};

describe('Data Streams - Project level max retention', () => {
const { httpSetup, httpRequestsMockHelpers } = setupEnvironment();
let testBed: DataStreamsTabTestBed;
jest.spyOn(breadcrumbService, 'setBreadcrumbs');

const notificationsServiceMock = notificationServiceMock.createStartContract();

beforeEach(async () => {
const {
setLoadIndicesResponse,
setLoadDataStreamsResponse,
setLoadDataStreamResponse,
setLoadTemplateResponse,
setLoadTemplatesResponse,
} = httpRequestsMockHelpers;

setLoadIndicesResponse([
createDataStreamBackingIndex('data-stream-index', 'dataStream1'),
createNonDataStreamIndex('non-data-stream-index'),
]);

const dataStreamForDetailPanel = createDataStreamPayload({
name: 'dataStream1',
storageSize: '5b',
storageSizeBytes: 5,
// metering API mock
meteringStorageSize: '156kb',
meteringStorageSizeBytes: 156000,
meteringDocsCount: 10000,
});

setLoadDataStreamsResponse([
dataStreamForDetailPanel,
createDataStreamPayload({
name: 'dataStream2',
storageSize: '1kb',
storageSizeBytes: 1000,
// metering API mock
meteringStorageSize: '156kb',
meteringStorageSizeBytes: 156000,
meteringDocsCount: 10000,
lifecycle: {
enabled: true,
data_retention: '7d',
effective_retention: '5d',
globalMaxRetention: '20d',
retention_determined_by: MAX_DATA_RETENTION,
},
}),
]);

setLoadDataStreamResponse(dataStreamForDetailPanel.name, dataStreamForDetailPanel);

const indexTemplate = fixtures.getTemplate({ name: 'indexTemplate' });
setLoadTemplatesResponse({ templates: [indexTemplate], legacyTemplates: [] });
setLoadTemplateResponse(indexTemplate.name, indexTemplate);

notificationService.setup(notificationsServiceMock);
testBed = await setup(httpSetup, {
history: createMemoryHistory(),
services: {
notificationService,
},
config: {
enableProjectLevelRetentionChecks: true,
},
});
await act(async () => {
testBed.actions.goToDataStreamsList();
});
testBed.component.update();
});

it('Should show error when retention value is bigger than project level retention', async () => {
const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers;

const ds1 = createDataStreamPayload({
name: 'dataStream1',
lifecycle: {
enabled: true,
data_retention: '25d',
effective_retention: '25d',
retention_determined_by: MAX_DATA_RETENTION,
globalMaxRetention: '20d',
},
});

setLoadDataStreamsResponse([ds1]);
setLoadDataStreamResponse(ds1.name, ds1);

testBed = await setup(httpSetup, {
history: createMemoryHistory(),
url: urlServiceMock,
config: {
enableProjectLevelRetentionChecks: true,
},
});
await act(async () => {
testBed.actions.goToDataStreamsList();
});
testBed.component.update();

const { actions } = testBed;

await actions.clickNameAt(0);
actions.clickEditDataRetentionButton();

expect(testBed.form.getErrorsMessages().length).toBeGreaterThan(0);
});
});
2 changes: 2 additions & 0 deletions x-pack/plugins/index_management/common/types/data_streams.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ export type DataStreamIndexFromEs = IndicesDataStreamIndex;
export type Health = 'green' | 'yellow' | 'red';

export interface EnhancedDataStreamFromEs extends IndicesDataStream {
global_max_retention?: string;
store_size?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size'];
store_size_bytes?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size_bytes'];
maximum_timestamp?: IndicesDataStreamsStatsDataStreamsStatsItem['maximum_timestamp'];
Expand Down Expand Up @@ -68,6 +69,7 @@ export interface DataStream {
enabled?: boolean;
effective_retention?: string;
retention_determined_by?: string;
globalMaxRetention?: string;
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ export interface AppDependencies {
editableIndexSettings: 'all' | 'limited';
enableMappingsSourceFieldSection: boolean;
enableTogglingDataRetention: boolean;
enableProjectLevelRetentionChecks: boolean;
enableSemanticText: boolean;
};
history: ScopedHistory;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { Dispatch, SetStateAction, useEffect, useState } from 'react';

function parseJsonOrDefault<Obj>(value: string | null, defaultValue: Obj): Obj {
if (!value) {
return defaultValue;
}
try {
return JSON.parse(value) as Obj;
} catch (e) {
return defaultValue;
}
}

export function useStateWithLocalStorage<State>(
key: string,
defaultState: State
): [State, Dispatch<SetStateAction<State>>] {
const storageState = localStorage.getItem(key);
const [state, setState] = useState<State>(parseJsonOrDefault<State>(storageState, defaultState));
useEffect(() => {
localStorage.setItem(key, JSON.stringify(state));
}, [key, state]);
return [state, setState];
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ import { i18n } from '@kbn/i18n';
import {
EuiFlexGroup,
EuiFlexItem,
EuiSwitch,
EuiText,
EuiIconTip,
EuiSpacer,
EuiPageSection,
EuiEmptyPrompt,
EuiCallOut,
EuiButton,
EuiLink,
} from '@elastic/eui';
import { ScopedHistory } from '@kbn/core/public';
Expand All @@ -40,8 +40,10 @@ import { documentationService } from '../../../services/documentation';
import { DataStreamTable } from './data_stream_table';
import { DataStreamDetailPanel } from './data_stream_detail_panel';
import { filterDataStreams, isSelectedDataStreamHidden } from '../../../lib/data_streams';
import { FilterListButton, Filters } from '../components';
import { Filters } from '../components';
import { useStateWithLocalStorage } from '../../../hooks/use_state_with_localstorage';

const SHOW_PROJECT_LEVEL_RETENTION = 'showProjectLevelRetention';
export type DataStreamFilterName = 'managed' | 'hidden';
interface MatchParams {
dataStreamName?: string;
Expand All @@ -58,8 +60,9 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
const decodedDataStreamName = attemptToURIDecode(dataStreamName);

const {
config: { enableProjectLevelRetentionChecks },
core: { getUrlForApp, executionContext },
plugins: { isFleetEnabled },
plugins: { isFleetEnabled, cloud },
} = useAppContext();

useExecutionContext(executionContext, {
Expand All @@ -81,6 +84,9 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
includeStats: isIncludeStatsChecked,
});

const [projectLevelRetentionCallout, setprojectLevelRetentionCallout] =
useStateWithLocalStorage<boolean>(SHOW_PROJECT_LEVEL_RETENTION, true);

const [filters, setFilters] = useState<Filters<DataStreamFilterName>>({
managed: {
name: i18n.translate('xpack.idxMgmt.dataStreamList.viewManagedLabel', {
Expand Down Expand Up @@ -125,7 +131,7 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
return (
<EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
<EuiFlexItem>
<EuiText color="subdued">
<EuiText color="subdued" css={{ maxWidth: '80%' }}>
<FormattedMessage
id="xpack.idxMgmt.dataStreamList.dataStreamsDescription"
defaultMessage="Data streams store time-series data across multiple indices and can be created from index templates. {learnMoreLink}"
Expand All @@ -146,38 +152,16 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
</EuiText>
</EuiFlexItem>

<EuiFlexItem grow={false}>
<EuiFlexGroup gutterSize="s">
<EuiFlexItem grow={false}>
<EuiSwitch
label={i18n.translate(
'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchLabel',
{
defaultMessage: 'Include stats',
}
)}
checked={isIncludeStatsChecked}
onChange={(e) => setIsIncludeStatsChecked(e.target.checked)}
data-test-subj="includeStatsSwitch"
/>
</EuiFlexItem>

<EuiFlexItem grow={false}>
<EuiIconTip
content={i18n.translate(
'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchToolTip',
{
defaultMessage: 'Including stats can increase reload times',
}
)}
position="top"
{enableProjectLevelRetentionChecks && (
<EuiFlexItem grow={false}>
<EuiLink href={cloud?.deploymentUrl} target="_blank">
<FormattedMessage
id="xpack.idxMgmt.dataStreamList.projectlevelRetention.linkText"
defaultMessage="Project data retention"
/>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<FilterListButton<DataStreamFilterName> filters={filters} onChange={setFilters} />
</EuiFlexItem>
</EuiLink>
</EuiFlexItem>
)}
</EuiFlexGroup>
);
};
Expand Down Expand Up @@ -276,6 +260,37 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
activateHiddenFilter(isSelectedDataStreamHidden(dataStreams!, decodedDataStreamName));
content = (
<EuiPageSection paddingSize="none" data-test-subj="dataStreamList">
{enableProjectLevelRetentionChecks && projectLevelRetentionCallout && (
<>
<EuiCallOut
onDismiss={() => setprojectLevelRetentionCallout(false)}
data-test-subj="projectLevelRetentionCallout"
title={i18n.translate(
'xpack.idxMgmt.dataStreamList.projectLevelRetentionCallout.titleText',
{
defaultMessage:
'You can now configure data stream retention settings for your entire project',
}
)}
>
<p>
<FormattedMessage
id="xpack.idxMgmt.dataStreamList.projectLevelRetentionCallout.descriptionText"
defaultMessage="Optionally define a maximum and default retention period to manage your compliance and storage size needs."
/>
</p>

<EuiButton href={cloud?.deploymentUrl} fill data-test-subj="cloudLinkButton">
<FormattedMessage
id="xpack.idxMgmt.dataStreamList.projectLevelRetentionCallout.buttonText"
defaultMessage="Get started"
/>
</EuiButton>
</EuiCallOut>
<EuiSpacer size="m" />
</>
)}

{renderHeader()}
<EuiSpacer size="l" />

Expand All @@ -287,8 +302,11 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
}
dataStreams={filteredDataStreams}
reload={reload}
viewFilters={filters}
onViewFilterChange={setFilters}
history={history as ScopedHistory}
includeStats={isIncludeStatsChecked}
setIncludeStats={setIsIncludeStatsChecked}
/>
</EuiPageSection>
);
Expand Down
Loading

0 comments on commit 47f11dd

Please sign in to comment.