From cc84826c52006ccd447e3a3852758d3a76fc46c4 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 16 Oct 2019 09:00:15 -0600 Subject: [PATCH 1/2] Add SLM support to xpack usage and info APIs This is a backport of #48096 This adds the missing xpack usage and info information into the `/_xpack` and `/_xpack/usage` APIs. The output now looks like: ``` GET /_xpack/usage { ... "slm" : { "available" : true, "enabled" : true, "policy_count" : 1, "policy_stats" : { "retention_runs" : 0, ... } } ``` and ``` GET /_xpack { ... "features" : { ... "slm" : { "available" : true, "enabled" : true }, ... } } ``` Relates to #43663 --- docs/reference/rest-api/info.asciidoc | 4 ++ .../xpack/core/XPackClientPlugin.java | 14 ++-- .../elasticsearch/xpack/core/XPackField.java | 2 + .../xpack/core/slm/SLMFeatureSetUsage.java | 72 +++++++++++++++++++ .../xpack/slm/SnapshotLifecycleRestIT.java | 71 ++++++++++++++++++ .../xpack/ilm/IndexLifecycle.java | 2 + .../xpack/slm/SLMFeatureSet.java | 61 ++++++++++++++++ 7 files changed, 221 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java create mode 100644 x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SLMFeatureSet.java diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index 9aae2c7331e7a..a07a73585f47d 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -111,6 +111,10 @@ Example response: "available" : true, "enabled" : false }, + "slm" : { + "available" : true, + "enabled" : true + }, "spatial" : { "available" : true, "enabled" : true diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index ba0a77cd0542b..e868296f9f78b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -38,10 +38,10 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; +import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; -import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.enrich.action.DeleteEnrichPolicyAction; import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction; @@ -152,12 +152,12 @@ import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.Recall; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.ScoreByThresholdResult; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.SoftClassificationMetric; -import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModel; -import org.elasticsearch.xpack.core.ml.inference.trainedmodel.tree.Tree; import org.elasticsearch.xpack.core.ml.inference.preprocessing.FrequencyEncoding; import org.elasticsearch.xpack.core.ml.inference.preprocessing.OneHotEncoding; import org.elasticsearch.xpack.core.ml.inference.preprocessing.PreProcessor; import org.elasticsearch.xpack.core.ml.inference.preprocessing.TargetMeanEncoding; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModel; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.tree.Tree; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage; import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage; @@ -204,6 +204,7 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges; import org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport; +import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction; import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction; @@ -223,10 +224,10 @@ import org.elasticsearch.xpack.core.transform.action.PutTransformAction; import org.elasticsearch.xpack.core.transform.action.StartTransformAction; import org.elasticsearch.xpack.core.transform.action.StopTransformAction; -import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; -import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformState; +import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction; import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction; import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage; @@ -544,6 +545,9 @@ public List getNamedWriteables() { // ILM new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE, IndexLifecycleFeatureSetUsage::new), + // SLM + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SNAPSHOT_LIFECYCLE, + SLMFeatureSetUsage::new), // ILM - Custom Metadata new NamedWriteableRegistry.Entry(MetaData.Custom.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata::new), new NamedWriteableRegistry.Entry(NamedDiff.class, IndexLifecycleMetadata.TYPE, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index ae4050bee7b45..11a851eb103fb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -33,6 +33,8 @@ public final class XPackField { public static final String ROLLUP = "rollup"; /** Name constant for the index lifecycle feature. */ public static final String INDEX_LIFECYCLE = "ilm"; + /** Name constant for the snapshot lifecycle management feature. */ + public static final String SNAPSHOT_LIFECYCLE = "slm"; /** Name constant for the CCR feature. */ public static final String CCR = "ccr"; /** Name constant for the transform feature. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java new file mode 100644 index 0000000000000..587cc2896aa35 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.slm; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.slm.SnapshotLifecycleStats; + +import java.io.IOException; +import java.util.Objects; + +public class SLMFeatureSetUsage extends XPackFeatureSet.Usage { + @Nullable + private final SnapshotLifecycleStats slmStats; + + public SLMFeatureSetUsage(StreamInput in) throws IOException { + super(in); + this.slmStats = in.readOptionalWriteable(SnapshotLifecycleStats::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalWriteable(this.slmStats); + } + + public SLMFeatureSetUsage(boolean available, boolean enabled, @Nullable SnapshotLifecycleStats slmStats) { + super(XPackField.SNAPSHOT_LIFECYCLE, available, enabled); + this.slmStats = slmStats; + } + + public SnapshotLifecycleStats getStats() { + return this.slmStats; + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + if (slmStats != null) { + builder.field("policy_count", slmStats.getMetrics().size()); + builder.field("policy_stats", slmStats); + } + } + + @Override + public int hashCode() { + return Objects.hash(available, enabled, slmStats); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + SLMFeatureSetUsage other = (SLMFeatureSetUsage) obj; + return Objects.equals(available, other.available) && + Objects.equals(enabled, other.enabled) && + Objects.equals(slmStats, other.slmStats); + } + +} diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java index 645fe458d0ac6..796612bcb8605 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java @@ -49,6 +49,7 @@ import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryItem.DELETE_OPERATION; import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryStore.SLM_HISTORY_INDEX_PREFIX; import static org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT.getStepKeyForIndex; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -422,6 +423,76 @@ public void testBasicTimeBasedRetenion() throws Exception { } } + @SuppressWarnings("unchecked") + public void testSLMXpackInfo() { + Map features = (Map) getLocation("/_xpack").get("features"); + assertNotNull(features); + Map slm = (Map) features.get("slm"); + assertNotNull(slm); + assertTrue((boolean) slm.get("available")); + assertTrue((boolean) slm.get("enabled")); + } + + @SuppressWarnings("unchecked") + public void testSLMXpackUsage() throws Exception { + Map slm = (Map) getLocation("/_xpack/usage").get("slm"); + assertNotNull(slm); + assertTrue((boolean) slm.get("available")); + assertTrue((boolean) slm.get("enabled")); + assertThat(slm.get("policy_count"), anyOf(equalTo(null), equalTo(0))); + + // Create a snapshot repo + initializeRepo("repo"); + // Create a policy with a retention period of 1 millisecond + createSnapshotPolicy("policy", "snap", "1 2 3 4 5 ?", "repo", "*", true, + new SnapshotRetentionConfiguration(TimeValue.timeValueMillis(1), null, null)); + final String snapshotName = executePolicy("policy"); + + // Check that the executed snapshot is created + assertBusy(() -> { + try { + logger.info("--> checking for snapshot creation..."); + Response response = client().performRequest(new Request("GET", "/_snapshot/repo/" + snapshotName)); + Map snapshotResponseMap; + try (InputStream is = response.getEntity().getContent()) { + snapshotResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true); + } + assertThat(snapshotResponseMap.size(), greaterThan(0)); + } catch (ResponseException e) { + fail("expected snapshot to exist but it does not: " + EntityUtils.toString(e.getResponse().getEntity())); + } + }); + + // Wait for stats to be updated + assertBusy(() -> { + logger.info("--> checking for stats to be updated..."); + Map stats = getSLMStats(); + Map policyStats = policyStatsAsMap(stats); + Map policyIdStats = (Map) policyStats.get("policy"); + assertNotNull(policyIdStats); + }); + + slm = (Map) getLocation("/_xpack/usage").get("slm"); + assertNotNull(slm); + assertTrue((boolean) slm.get("available")); + assertTrue((boolean) slm.get("enabled")); + assertThat("got: " + slm, slm.get("policy_count"), equalTo(1)); + assertNotNull(slm.get("policy_stats")); + } + + public Map getLocation(String path) { + try { + Response executeRepsonse = client().performRequest(new Request("GET", path)); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, EntityUtils.toByteArray(executeRepsonse.getEntity()))) { + return parser.map(); + } + } catch (Exception e) { + fail("failed to execute GET request to " + path + " - got: " + e); + throw new RuntimeException(e); + } + } + /** * Execute the given policy and return the generated snapshot name */ diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 28c945e36e766..20233812a3cde 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -94,6 +94,7 @@ import org.elasticsearch.xpack.ilm.action.TransportRetryAction; import org.elasticsearch.xpack.ilm.action.TransportStartILMAction; import org.elasticsearch.xpack.ilm.action.TransportStopILMAction; +import org.elasticsearch.xpack.slm.SLMFeatureSet; import org.elasticsearch.xpack.slm.SnapshotLifecycleService; import org.elasticsearch.xpack.slm.SnapshotLifecycleTask; import org.elasticsearch.xpack.slm.SnapshotRetentionService; @@ -158,6 +159,7 @@ public Collection createGuiceModules() { } modules.add(b -> XPackPlugin.bindFeatureSet(b, IndexLifecycleFeatureSet.class)); + modules.add(b -> XPackPlugin.bindFeatureSet(b, SLMFeatureSet.class)); return modules; } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SLMFeatureSet.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SLMFeatureSet.java new file mode 100644 index 0000000000000..8d565f020c25e --- /dev/null +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SLMFeatureSet.java @@ -0,0 +1,61 @@ +package org.elasticsearch.xpack.slm; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage; +import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; + +import java.util.Map; + +public class SLMFeatureSet implements XPackFeatureSet { + + private final boolean enabled; + private final XPackLicenseState licenseState; + private ClusterService clusterService; + + @Inject + public SLMFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) { + this.clusterService = clusterService; + this.enabled = XPackSettings.SNAPSHOT_LIFECYCLE_ENABLED.get(settings); + this.licenseState = licenseState; + } + + @Override + public String name() { + return XPackField.SNAPSHOT_LIFECYCLE; + } + + @Override + public boolean available() { + return licenseState != null && licenseState.isIndexLifecycleAllowed(); + } + + @Override + public boolean enabled() { + return enabled; + } + + @Override + public Map nativeCodeInfo() { + return null; + } + + @Override + public void usage(ActionListener listener) { + final ClusterState state = clusterService.state(); + boolean available = licenseState.isIndexLifecycleAllowed(); + final SnapshotLifecycleMetadata slmMeta = state.metaData().custom(SnapshotLifecycleMetadata.TYPE); + final SLMFeatureSetUsage usage = new SLMFeatureSetUsage(available, enabled, + slmMeta == null ? null : slmMeta.getStats()); + listener.onResponse(usage); + } + +} From f43381102dedb2a1a31bf610cfbd9bc33a9c56b3 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 16 Oct 2019 13:21:22 -0600 Subject: [PATCH 2/2] Fix missing license --- .../java/org/elasticsearch/xpack/slm/SLMFeatureSet.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SLMFeatureSet.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SLMFeatureSet.java index 8d565f020c25e..23796984d6505 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SLMFeatureSet.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SLMFeatureSet.java @@ -1,3 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + package org.elasticsearch.xpack.slm; import org.elasticsearch.action.ActionListener;