Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add SLM support to xpack usage and info APIs #48150

Merged
merged 4 commits into from
Oct 17, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/reference/rest-api/info.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,10 @@ Example response:
"available" : true,
"enabled" : false
},
"slm" : {
"available" : true,
"enabled" : true
},
"spatial" : {
"available" : true,
"enabled" : true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@
import org.elasticsearch.transport.Transport;
import org.elasticsearch.xpack.core.action.XPackInfoAction;
import org.elasticsearch.xpack.core.action.XPackUsageAction;
import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
import org.elasticsearch.xpack.core.enrich.action.DeleteEnrichPolicyAction;
import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction;
Expand Down Expand Up @@ -152,12 +152,12 @@
import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.Recall;
import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.ScoreByThresholdResult;
import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.SoftClassificationMetric;
import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModel;
import org.elasticsearch.xpack.core.ml.inference.trainedmodel.tree.Tree;
import org.elasticsearch.xpack.core.ml.inference.preprocessing.FrequencyEncoding;
import org.elasticsearch.xpack.core.ml.inference.preprocessing.OneHotEncoding;
import org.elasticsearch.xpack.core.ml.inference.preprocessing.PreProcessor;
import org.elasticsearch.xpack.core.ml.inference.preprocessing.TargetMeanEncoding;
import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModel;
import org.elasticsearch.xpack.core.ml.inference.trainedmodel.tree.Tree;
import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage;
Expand Down Expand Up @@ -204,6 +204,7 @@
import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
import org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport;
import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage;
import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction;
import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction;
Expand All @@ -223,10 +224,10 @@
import org.elasticsearch.xpack.core.transform.action.PutTransformAction;
import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
import org.elasticsearch.xpack.core.transform.action.StopTransformAction;
import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
import org.elasticsearch.xpack.core.transform.transforms.TransformState;
import org.elasticsearch.xpack.core.transform.transforms.SyncConfig;
import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformState;
import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction;
import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction;
import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage;
Expand Down Expand Up @@ -544,6 +545,9 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
// ILM
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
IndexLifecycleFeatureSetUsage::new),
// SLM
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SNAPSHOT_LIFECYCLE,
SLMFeatureSetUsage::new),
// ILM - Custom Metadata
new NamedWriteableRegistry.Entry(MetaData.Custom.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, IndexLifecycleMetadata.TYPE,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ public final class XPackField {
public static final String ROLLUP = "rollup";
/** Name constant for the index lifecycle feature. */
public static final String INDEX_LIFECYCLE = "ilm";
/** Name constant for the snapshot lifecycle management feature. */
public static final String SNAPSHOT_LIFECYCLE = "slm";
/** Name constant for the CCR feature. */
public static final String CCR = "ccr";
/** Name constant for the transform feature. */
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.core.slm;

import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.XPackFeatureSet;
import org.elasticsearch.xpack.core.XPackField;
import org.elasticsearch.xpack.slm.SnapshotLifecycleStats;

import java.io.IOException;
import java.util.Objects;

public class SLMFeatureSetUsage extends XPackFeatureSet.Usage {
@Nullable
private final SnapshotLifecycleStats slmStats;

public SLMFeatureSetUsage(StreamInput in) throws IOException {
super(in);
this.slmStats = in.readOptionalWriteable(SnapshotLifecycleStats::new);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeOptionalWriteable(this.slmStats);
}

public SLMFeatureSetUsage(boolean available, boolean enabled, @Nullable SnapshotLifecycleStats slmStats) {
super(XPackField.SNAPSHOT_LIFECYCLE, available, enabled);
this.slmStats = slmStats;
}

public SnapshotLifecycleStats getStats() {
return this.slmStats;
}

@Override
protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
super.innerXContent(builder, params);
if (slmStats != null) {
builder.field("policy_count", slmStats.getMetrics().size());
builder.field("policy_stats", slmStats);
}
}

@Override
public int hashCode() {
return Objects.hash(available, enabled, slmStats);
}

@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
SLMFeatureSetUsage other = (SLMFeatureSetUsage) obj;
return Objects.equals(available, other.available) &&
Objects.equals(enabled, other.enabled) &&
Objects.equals(slmStats, other.slmStats);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryItem.DELETE_OPERATION;
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryStore.SLM_HISTORY_INDEX_PREFIX;
import static org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT.getStepKeyForIndex;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
Expand Down Expand Up @@ -422,6 +423,76 @@ public void testBasicTimeBasedRetenion() throws Exception {
}
}

@SuppressWarnings("unchecked")
public void testSLMXpackInfo() {
Map<String, Object> features = (Map<String, Object>) getLocation("/_xpack").get("features");
assertNotNull(features);
Map<String, Object> slm = (Map<String, Object>) features.get("slm");
assertNotNull(slm);
assertTrue((boolean) slm.get("available"));
assertTrue((boolean) slm.get("enabled"));
}

@SuppressWarnings("unchecked")
public void testSLMXpackUsage() throws Exception {
Map<String, Object> slm = (Map<String, Object>) getLocation("/_xpack/usage").get("slm");
assertNotNull(slm);
assertTrue((boolean) slm.get("available"));
assertTrue((boolean) slm.get("enabled"));
assertThat(slm.get("policy_count"), anyOf(equalTo(null), equalTo(0)));

// Create a snapshot repo
initializeRepo("repo");
// Create a policy with a retention period of 1 millisecond
createSnapshotPolicy("policy", "snap", "1 2 3 4 5 ?", "repo", "*", true,
new SnapshotRetentionConfiguration(TimeValue.timeValueMillis(1), null, null));
final String snapshotName = executePolicy("policy");

// Check that the executed snapshot is created
assertBusy(() -> {
try {
logger.info("--> checking for snapshot creation...");
Response response = client().performRequest(new Request("GET", "/_snapshot/repo/" + snapshotName));
Map<String, Object> snapshotResponseMap;
try (InputStream is = response.getEntity().getContent()) {
snapshotResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
}
assertThat(snapshotResponseMap.size(), greaterThan(0));
} catch (ResponseException e) {
fail("expected snapshot to exist but it does not: " + EntityUtils.toString(e.getResponse().getEntity()));
}
});

// Wait for stats to be updated
assertBusy(() -> {
logger.info("--> checking for stats to be updated...");
Map<String, Object> stats = getSLMStats();
Map<String, Object> policyStats = policyStatsAsMap(stats);
Map<String, Object> policyIdStats = (Map<String, Object>) policyStats.get("policy");
assertNotNull(policyIdStats);
});

slm = (Map<String, Object>) getLocation("/_xpack/usage").get("slm");
assertNotNull(slm);
assertTrue((boolean) slm.get("available"));
assertTrue((boolean) slm.get("enabled"));
assertThat("got: " + slm, slm.get("policy_count"), equalTo(1));
assertNotNull(slm.get("policy_stats"));
}

public Map<String, Object> getLocation(String path) {
try {
Response executeRepsonse = client().performRequest(new Request("GET", path));
try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, EntityUtils.toByteArray(executeRepsonse.getEntity()))) {
return parser.map();
}
} catch (Exception e) {
fail("failed to execute GET request to " + path + " - got: " + e);
throw new RuntimeException(e);
}
}

/**
* Execute the given policy and return the generated snapshot name
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
import org.elasticsearch.xpack.ilm.action.TransportRetryAction;
import org.elasticsearch.xpack.ilm.action.TransportStartILMAction;
import org.elasticsearch.xpack.ilm.action.TransportStopILMAction;
import org.elasticsearch.xpack.slm.SLMFeatureSet;
import org.elasticsearch.xpack.slm.SnapshotLifecycleService;
import org.elasticsearch.xpack.slm.SnapshotLifecycleTask;
import org.elasticsearch.xpack.slm.SnapshotRetentionService;
Expand Down Expand Up @@ -158,6 +159,7 @@ public Collection<Module> createGuiceModules() {
}

modules.add(b -> XPackPlugin.bindFeatureSet(b, IndexLifecycleFeatureSet.class));
modules.add(b -> XPackPlugin.bindFeatureSet(b, SLMFeatureSet.class));

return modules;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.slm;

import org.elasticsearch.action.ActionListener;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.xpack.core.XPackFeatureSet;
import org.elasticsearch.xpack.core.XPackField;
import org.elasticsearch.xpack.core.XPackSettings;
import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage;
import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;

import java.util.Map;

public class SLMFeatureSet implements XPackFeatureSet {

private final boolean enabled;
private final XPackLicenseState licenseState;
private ClusterService clusterService;

@Inject
public SLMFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) {
this.clusterService = clusterService;
this.enabled = XPackSettings.SNAPSHOT_LIFECYCLE_ENABLED.get(settings);
this.licenseState = licenseState;
}

@Override
public String name() {
return XPackField.SNAPSHOT_LIFECYCLE;
}

@Override
public boolean available() {
return licenseState != null && licenseState.isIndexLifecycleAllowed();
}

@Override
public boolean enabled() {
return enabled;
}

@Override
public Map<String, Object> nativeCodeInfo() {
return null;
}

@Override
public void usage(ActionListener<Usage> listener) {
final ClusterState state = clusterService.state();
boolean available = licenseState.isIndexLifecycleAllowed();
final SnapshotLifecycleMetadata slmMeta = state.metaData().custom(SnapshotLifecycleMetadata.TYPE);
final SLMFeatureSetUsage usage = new SLMFeatureSetUsage(available, enabled,
slmMeta == null ? null : slmMeta.getStats());
listener.onResponse(usage);
}

}