[WIP] Index lifecycle (#25071)

* Index lifecycle management wizard * Adding index lifecycle management files * Updates * Fix errors and add flyout for node details * New diff tool * Scroll to change for review diff * Some feedback on copy * Updating copy, moving components around and fixing bugs with the diff view * Add logic to auto enable a phase when something is set * redesign ilm * Adding server api tests * Removing debug and some tweaks from dave's work * Conditionally show this message * Policy selection cleanup * Updates for better UX * [COPYEDIT | ILM] Copyedit of text in index lifecycle management UI * Use better default text * Remove debug * Adding readme and comments * Update readme * Do not need this anymore * Remove debug or commented out code * Remove these - they are in the tests PR * Toggle system indices * Aliases are not defined here anymore * Handle rollover better in warm phase and remove from cold,delete * adding learn more link component and switching over to using that * fixing UI issue when no policies exist * various fixes * some cleanup * moving number_of_replicas due to API change * modifying some messaging * fixing typo * fixing some diff issues and not adding sattr_name if none chosen * making write alias required for template step and making necessary API changes * removing alias definition from template patching as it needs to be per index * some copy edits for clarity * fixing issue with editing existing policy when rollover starts the warm phase * addressing PR feedback on server side code * addressing PR feedback * removing additional spaces from findMatchingNodes call * changing template to index template in one more place * fixing issue with error message showing when bootstrap is successful * fixing node options for warm and cold phase * adding seconds to duration fields to match what ES supports * changing icon for enabled steps so it does not look like an error indicator * adjusting icon color for enabled lifecycle steps * fixing issue with editing an existing policy with warm phase on rollover enabled * fixing issue with default unit for age dropdowns * fixing issues with shrink action serialization and deserialization * fixing issue with deserialization of ES policy for shrink * removing shrink option from UI when primary shard count is not greater than 1 for hot phase * going straight to create policy when no policies exist * improving lifecycle policy selection * adding active badge instead of checkmark for active lifecycle policy phases * some cleanup of unneeded properties and only showing save as new when it is appropriate * removing stray fullWidth attribute * adding missing minimum for replica count for warm phase * adding scroll to top for review step * fixing issue with start warm phase after rollover introduced by time representaiton change from ES * making shrink options not show for primary shard count of 1 as you can't shrink in that situation * fixing issue with editing existing policy and saveAsNew * bare bones policy table implementation * implementing delete policy behavior * fixing sorting and paging * fixing policy table title * rudimentary navigation flow * fixing delete * Index lifecycle management wizard (#21925) * Index lifecycle management wizard * Adding index lifecycle management files * Updates * Fix errors and add flyout for node details * New diff tool * Scroll to change for review diff * Some feedback on copy * Updating copy, moving components around and fixing bugs with the diff view * Add logic to auto enable a phase when something is set * redesign ilm * Adding server api tests * Removing debug and some tweaks from dave's work * Conditionally show this message * Policy selection cleanup * Updates for better UX * [COPYEDIT | ILM] Copyedit of text in index lifecycle management UI * Use better default text * Remove debug * Adding readme and comments * Update readme * Do not need this anymore * Remove debug or commented out code * Remove these - they are in the tests PR * Toggle system indices * Aliases are not defined here anymore * Handle rollover better in warm phase and remove from cold,delete * adding learn more link component and switching over to using that * fixing UI issue when no policies exist * various fixes * some cleanup * moving number_of_replicas due to API change * modifying some messaging * fixing typo * fixing some diff issues and not adding sattr_name if none chosen * making write alias required for template step and making necessary API changes * removing alias definition from template patching as it needs to be per index * some copy edits for clarity * fixing issue with editing existing policy when rollover starts the warm phase * addressing PR feedback on server side code * addressing PR feedback * removing additional spaces from findMatchingNodes call * changing template to index template in one more place * fixing issue with error message showing when bootstrap is successful * fixing node options for warm and cold phase * adding seconds to duration fields to match what ES supports * changing icon for enabled steps so it does not look like an error indicator * adjusting icon color for enabled lifecycle steps * fixing issue with editing an existing policy with warm phase on rollover enabled * fixing issue with default unit for age dropdowns * fixing issues with shrink action serialization and deserialization * fixing issue with deserialization of ES policy for shrink * removing shrink option from UI when primary shard count is not greater than 1 for hot phase * going straight to create policy when no policies exist * improving lifecycle policy selection * adding active badge instead of checkmark for active lifecycle policy phases * some cleanup of unneeded properties and only showing save as new when it is appropriate * removing stray fullWidth attribute * adding missing minimum for replica count for warm phase * adding scroll to top for review step * fixing issue with start warm phase after rollover introduced by time representaiton change from ES * making shrink options not show for primary shard count of 1 as you can't shrink in that situation * fixing issue with editing existing policy and saveAsNew * adjusting to changes in ES API * adding version and modified date to policies table * implementing new CRUD approach * simplified delete * cleanup edit_policy * removed wizard code * fixing issue with edit policy * fixing issue with closing delete confirmation modal * making max age and max size not mutually exclusive * removing names of covered indices from policy table * changing minimum_age to min_age * first pass at index lifecycle extensions * adding retry button for ilm covered index that is in error * first pass at index lifecycle banner * i18n work * more i18n * fixing issue with node attributes * removing console.log statements * fixing issue with deserializing number_of_shards for edit policy * defaulting shrink to false and fixing ui spacing issue * removing hot phase shard count from warm phase * scrolling to first error when user submits form for edit policy * disabling UI for index management when enabled is false in kibana.yml * disabling index lifecycle management when enabled is false in kibana.yml * extending index management filter to allow for searching fields * add support for filtering to indices with errors for index lifecycle management banner * i18n work * fixing error wrappers * fixing tests * adding view JSON for index lifecycle policy on edit screen * fixing label for i18n on policy JSON flyout * removing console.log statements * fixing tests
elastic · Nov 12, 2018 · 25cfc88 · 25cfc88
1 parent 77522a7
commit 25cfc88
Show file tree

Hide file tree

Showing 143 changed files with 6,532 additions and 72 deletions.
diff --git a/x-pack/index.js b/x-pack/index.js
@@ -21,6 +21,7 @@ import { apm } from './plugins/apm';
 import { licenseManagement } from './plugins/license_management';
 import { cloud } from './plugins/cloud';
 import { indexManagement } from './plugins/index_management';
+import { indexLifecycleManagement } from './plugins/index_lifecycle_management';
 import { consoleExtensions } from './plugins/console_extensions';
 import { spaces } from './plugins/spaces';
 import { notifications } from './plugins/notifications';
@@ -52,6 +53,7 @@ module.exports = function (kibana) {
     indexManagement(kibana),
     consoleExtensions(kibana),
     notifications(kibana),
+    indexLifecycleManagement(kibana),
     kueryAutocomplete(kibana),
     infra(kibana),
     rollup(kibana),

diff --git a/x-pack/plugins/index_lifecycle_management/README.md b/x-pack/plugins/index_lifecycle_management/README.md
@@ -0,0 +1,72 @@
+# Index lifecyle management
+
+## What is it
+-- TODO --
+
+## UI
+
+The UI currently consists of a single wizard, broken into three steps.
+
+### Step 1
+The first step involves choosing the index template in which the created/selected policy will be applied.
+Then, it lets the user tweak configuration options on this template including shard and replica count as well as allocation rules.
+
+### Step 2
+The second step lets the user choose which policy they want to apply to the selected index template. They can choose a new one or select an existing one. Either way, after selection, they will see configuration options for the policy itself. This includes configuration for the hot, warm, cold, and delete phase.
+
+### Step 3
+The third and last step lets the user name their policy and also see the affected indices and index templates. These indices and index templates are what will be affected once the user saves the work done in the wizard (This includes changes to the index template itself which will change indices created from the template and also changes to a policy that is attached to another index template). The user can also see a visual diff of what will change in the index template. Then, the user clicks the Save button and blamo!
+
+## UI Architecture
+
+The UI is built on React and Redux.
+
+### Redux
+
+The redux store consists of a few top level attributes:
+```
+indexTemplate
+nodes
+policies
+general
+```
+
+The logic behind the store is separate into four main concerns:
+1) reducers/
+2) actions/
+3) selectors/
+4) middleware/
+
+The reducers and actions are pretty standard redux, so no need to discuss much there.
+
+### Selectors
+
+The selectors showcase how we access any stateful data. All access comes through selectors so if there are any changes required to the state tree, we only need to update the reducers and selectors.
+
+#### Middleware
+
+The middleware folder contains specific pieces of state logic we need to handle side effects of certain state changing.
+
+One example is the `auto_enable_phase.js` middleware. By default, the warm, cold and delete phases are disabled. However, the user can expand the section in the UI and edit configuration without needing to enable/disable the phase. Ideally, once the user edits any configuration piece within a phase, we _assume_ they want that phase enabled so this middleware will detect a change in a phase and enable if it is not enabled already.
+
+#### Generic phase data
+
+Each of our four phases have some similar and some unique configuration options. Instead of making each individual phase a specific action for that phase, the code is written more generically to capture any data change within a phase to a single action. Therefore, each phase component's configuration inputs will look similar, like: `setPhaseData(PHASE_ROLLOVER_MINIMUM_AGE_UNITS, e.target.value)`. The top level container for each phase will handle automatically prefixing the `setPhaseData` prop with the right phase: ` setPhaseData: (key, value) => setPhaseData(PHASE_COLD, key, value),`.
+
+To complement this generic logic, there is a list of constants that are used to ensure the right pieces of data are changed. These are contained within `store/constants.js`
+
+### Diff View
+
+The third step of the wizard features a visual diff UI component which is custom to this feature. It is based off Ace/Brace and the custom code used to power is it lives in `lib/diff_ace_addons.js` and `lib/diff_tools.js`. The UI parts are in `sections/wizard/components/review/diff_view.js`. See those individual files for more detailed comments/explanations.
+
+### Validation
+
+Every step in the wizard features validation and will show error states after the user attempts to move to the next step assuming there are errors on the current page.
+
+This works by constantly revalidating the entire wizard state after each state change. This is technically optional as the method to trigger validation is manually called in each UI component that triggers a state change.
+
+It's important to note that the validation logic does not apply to a single step in the wizard, but will always validate the entire state tree. This helps prevent scenarios where a change in a step might invalidate a change in another step and we lose that validation state.
+
+Once a step change is initiated (like clicking Next Step), the current step is marked as able to see errors and will reject the change if there are errors. It will show a toast to the user that there are errors and make each error visible on the relevant UI control.
+
+As a way to consolidate showing these errors, there is a custom UI component called `ErrableFormRow` that wraps a `EuiFormRow` and it's child with the appropriate error states when appropriate.
diff --git a/x-pack/plugins/index_lifecycle_management/common/constants/base_path.js b/x-pack/plugins/index_lifecycle_management/common/constants/base_path.js
@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const BASE_PATH = '/management/elasticsearch/index_lifecycle_management/';
diff --git a/x-pack/plugins/index_lifecycle_management/common/constants/index.js b/x-pack/plugins/index_lifecycle_management/common/constants/index.js
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { PLUGIN } from './plugin';
+export { BASE_PATH } from './base_path';
diff --git a/x-pack/plugins/index_lifecycle_management/common/constants/plugin.js b/x-pack/plugins/index_lifecycle_management/common/constants/plugin.js
@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const PLUGIN = {
+  ID: 'index_lifecycle_management'
+};
diff --git a/x-pack/plugins/index_lifecycle_management/index.js b/x-pack/plugins/index_lifecycle_management/index.js
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { resolve } from 'path';
+import { registerTemplatesRoutes } from './server/routes/api/templates';
+import { registerNodesRoutes } from './server/routes/api/nodes';
+import { registerPoliciesRoutes } from './server/routes/api/policies';
+import { registerLifecycleRoutes } from './server/routes/api/lifecycle';
+import { registerIndexRoutes } from './server/routes/api/index';
+import { registerLicenseChecker } from './server/lib/register_license_checker';
+import { PLUGIN } from './common/constants';
+import { indexLifecycleDataEnricher } from './index_lifecycle_data';
+export function indexLifecycleManagement(kibana) {
+  return new kibana.Plugin({
+    id: PLUGIN.ID,
+    publicDir: resolve(__dirname, 'public'),
+    require: ['kibana', 'elasticsearch', 'xpack_main', 'index_management'],
+    uiExports: {
+      managementSections: ['plugins/index_lifecycle_management'],
+      injectDefaultVars(server) {
+        const config = server.config();
+        return {
+          indexLifecycleManagementUiEnabled: config.get(`${PLUGIN.ID}.enabled`)
+        };
+      },
+    },
+    init: function (server) {
+      registerLicenseChecker(server);
+      registerTemplatesRoutes(server);
+      registerNodesRoutes(server);
+      registerPoliciesRoutes(server);
+      registerLifecycleRoutes(server);
+      registerIndexRoutes(server);
+      if (
+        server.plugins.index_management &&
+        server.plugins.index_management.addIndexManagementDataEnricher
+      ) {
+        server.plugins.index_management.addIndexManagementDataEnricher(indexLifecycleDataEnricher);
+      }
+    },
+  });
+}
diff --git a/x-pack/plugins/index_lifecycle_management/index_lifecycle_data.js b/x-pack/plugins/index_lifecycle_management/index_lifecycle_data.js
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const indexLifecycleDataEnricher = async (indicesList, callWithRequest) => {
+  if (!indicesList || !indicesList.length) {
+    return;
+  }
+  const params = {
+    path: '/*/_ilm/explain',
+    method: 'GET',
+  };
+  const { indices: ilmIndicesData } = await callWithRequest('transport.request', params);
+  return indicesList.map(index => {
+    return {
+      ...index,
+      ilm: { ...(ilmIndicesData[index.name] || {}) },
+    };
+  });
+};
diff --git a/x-pack/plugins/index_lifecycle_management/public/api/index.js b/x-pack/plugins/index_lifecycle_management/public/api/index.js
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import chrome from 'ui/chrome';
+
+let httpClient;
+export const setHttpClient = (client) => {
+  httpClient = client;
+};
+const getHttpClient = () => {
+  return httpClient;
+};
+const apiPrefix = chrome.addBasePath('/api/index_lifecycle_management');
+
+export async function loadNodes(httpClient = getHttpClient()) {
+  const response = await httpClient.get(`${apiPrefix}/nodes/list`);
+  return response.data;
+}
+
+export async function loadNodeDetails(selectedNodeAttrs, httpClient = getHttpClient()) {
+  const response = await httpClient.get(`${apiPrefix}/nodes/${selectedNodeAttrs}/details`);
+  return response.data;
+}
+
+export async function loadIndexTemplates(httpClient = getHttpClient()) {
+  const response = await httpClient.get(`${apiPrefix}/templates`);
+  return response.data;
+}
+
+export async function loadIndexTemplate(templateName, httpClient = getHttpClient()) {
+  if (!templateName) {
+    return {};
+  }
+  const response = await httpClient.get(`${apiPrefix}/template/${templateName}`);
+  return response.data;
+}
+
+export async function loadPolicies(withIndices, httpClient = getHttpClient()) {
+  const response = await httpClient.get(`${apiPrefix}/policies${ withIndices ? '?withIndices=true' : ''}`);
+  return response.data;
+}
+
+export async function deletePolicy(policyName, httpClient = getHttpClient()) {
+  const response = await httpClient.delete(`${apiPrefix}/policies/${policyName}`);
+  return response.data;
+}
+
+export async function saveLifecycle(lifecycle, httpClient = getHttpClient()) {
+  const response = await httpClient.post(`${apiPrefix}/lifecycle`, { lifecycle });
+  return response.data;
+}
+
+
+export async function getAffectedIndices(indexTemplateName, policyName, httpClient = getHttpClient()) {
+  const path = policyName
+    ? `${apiPrefix}/indices/affected/${indexTemplateName}/${encodeURIComponent(policyName)}`
+    : `${apiPrefix}/indices/affected/${indexTemplateName}`;
+  const response = await httpClient.get(path);
+  return response.data;
+}
+export const retryLifecycleForIndex = async (indexNames, httpClient = getHttpClient()) => {
+  const response = await httpClient.post(`${apiPrefix}/index/retry`, { indexNames });
+  return response.data;
+};
diff --git a/x-pack/plugins/index_lifecycle_management/public/app.js b/x-pack/plugins/index_lifecycle_management/public/app.js
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { HashRouter, Switch, Route } from 'react-router-dom';
+import { EditPolicy } from './sections/edit_policy';
+import { PolicyTable } from './sections/policy_table';
+import { BASE_PATH } from '../common/constants';
+
+export const App = () => (
+  <HashRouter>
+    <Switch>
+      <Route exact path={`${BASE_PATH}policies`} component={PolicyTable}/>
+      <Route path={`${BASE_PATH}policies/edit/:policyName?`} component={EditPolicy}/>
+    </Switch>
+  </HashRouter>
+);
diff --git a/x-pack/plugins/index_lifecycle_management/public/components/active_badge.js b/x-pack/plugins/index_lifecycle_management/public/components/active_badge.js
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React from 'react';
+import { EuiBadge } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+export const ActiveBadge = () => {
+  return (
+    <EuiBadge className="eui-alignMiddle">
+      <FormattedMessage
+        id="xpack.indexLifecycleMgmt.activePhaseMessage"
+        defaultMessage="Active"
+      />
+    </EuiBadge>
+  );
+};
diff --git a/x-pack/plugins/index_lifecycle_management/public/components/index.js b/x-pack/plugins/index_lifecycle_management/public/components/index.js
@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { ActiveBadge } from './active_badge';
+export { LearnMoreLink } from './learn_more_link';
+export { PhaseErrorMessage } from './phase_error_message';