From 40de7de109587d59d46eb023c92fe7b1b92c59d1 Mon Sep 17 00:00:00 2001 From: Armin Braun <me@obrown.io> Date: Wed, 30 Sep 2020 13:32:54 +0200 Subject: [PATCH] Add Clone Snapshot Request Handling Scaffolding (#63037) Adds all the scaffolding for snapshot clone request handling and index-to-clone resolution to reduce the diff in #61839 to the bare essentials of the state machine changes for snapshot cloning and relevant tests and give us the opportunity to review the API in isolation. --- .../elasticsearch/action/ActionModule.java | 5 + .../snapshots/clone/CloneSnapshotAction.java | 33 ++++ .../snapshots/clone/CloneSnapshotRequest.java | 142 ++++++++++++++++++ .../clone/CloneSnapshotRequestBuilder.java | 65 ++++++++ .../clone/TransportCloneSnapshotAction.java | 75 +++++++++ .../client/ClusterAdminClient.java | 19 ++- .../client/support/AbstractClient.java | 18 +++ .../cluster/RestCloneSnapshotAction.java | 62 ++++++++ 8 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotAction.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCloneSnapshotAction.java diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index e2f7a65d6143f..96ff879c68682 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -64,6 +64,8 @@ import org.elasticsearch.action.admin.cluster.settings.TransportClusterUpdateSettingsAction; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction; import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction; +import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotAction; +import org.elasticsearch.action.admin.cluster.snapshots.clone.TransportCloneSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotAction; @@ -267,6 +269,7 @@ import org.elasticsearch.rest.action.admin.cluster.RestCancelTasksAction; import org.elasticsearch.rest.action.admin.cluster.RestCleanupRepositoryAction; import org.elasticsearch.rest.action.admin.cluster.RestClearVotingConfigExclusionsAction; +import org.elasticsearch.rest.action.admin.cluster.RestCloneSnapshotAction; import org.elasticsearch.rest.action.admin.cluster.RestClusterAllocationExplainAction; import org.elasticsearch.rest.action.admin.cluster.RestClusterGetSettingsAction; import org.elasticsearch.rest.action.admin.cluster.RestClusterHealthAction; @@ -522,6 +525,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg actions.register(GetSnapshotsAction.INSTANCE, TransportGetSnapshotsAction.class); actions.register(DeleteSnapshotAction.INSTANCE, TransportDeleteSnapshotAction.class); actions.register(CreateSnapshotAction.INSTANCE, TransportCreateSnapshotAction.class); + actions.register(CloneSnapshotAction.INSTANCE, TransportCloneSnapshotAction.class); actions.register(RestoreSnapshotAction.INSTANCE, TransportRestoreSnapshotAction.class); actions.register(SnapshotsStatusAction.INSTANCE, TransportSnapshotsStatusAction.class); @@ -665,6 +669,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) { registerHandler.accept(new RestCleanupRepositoryAction()); registerHandler.accept(new RestGetSnapshotsAction()); registerHandler.accept(new RestCreateSnapshotAction()); + registerHandler.accept(new RestCloneSnapshotAction()); registerHandler.accept(new RestRestoreSnapshotAction()); registerHandler.accept(new RestDeleteSnapshotAction()); registerHandler.accept(new RestSnapshotsStatusAction()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotAction.java new file mode 100644 index 0000000000000..e995469759bf0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotAction.java @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.clone; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedResponse; + +public final class CloneSnapshotAction extends ActionType<AcknowledgedResponse> { + + public static final CloneSnapshotAction INSTANCE = new CloneSnapshotAction(); + public static final String NAME = "cluster:admin/snapshot/clone"; + + private CloneSnapshotAction() { + super(NAME, AcknowledgedResponse::new); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java new file mode 100644 index 0000000000000..4d1eb0952e384 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java @@ -0,0 +1,142 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.clone; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class CloneSnapshotRequest extends MasterNodeRequest<CloneSnapshotRequest> implements IndicesRequest.Replaceable{ + + private final String repository; + + private final String source; + + private final String target; + + private String[] indices; + + private IndicesOptions indicesOptions = IndicesOptions.strictExpandHidden(); + + public CloneSnapshotRequest(StreamInput in) throws IOException { + super(in); + repository = in.readString(); + source = in.readString(); + target = in.readString(); + indices = in.readStringArray(); + indicesOptions = IndicesOptions.readIndicesOptions(in); + } + + /** + * Creates a clone snapshot request for cloning the given source snapshot's indices into the given target snapshot on the given + * repository. + * + * @param repository repository that source snapshot belongs to and that the target snapshot will be created in + * @param source source snapshot name + * @param target target snapshot name + * @param indices indices to clone from source to target + */ + public CloneSnapshotRequest(String repository, String source, String target, String[] indices) { + this.repository = repository; + this.source = source; + this.target = target; + this.indices = indices; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(repository); + out.writeString(source); + out.writeString(target); + out.writeStringArray(indices); + indicesOptions.writeIndicesOptions(out); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (source == null) { + validationException = addValidationError("source snapshot name is missing", null); + } + if (target == null) { + validationException = addValidationError("target snapshot name is missing", null); + } + if (repository == null) { + validationException = addValidationError("repository is missing", validationException); + } + if (indices == null) { + validationException = addValidationError("indices is null", validationException); + } else if (indices.length == 0) { + validationException = addValidationError("indices patterns are empty", validationException); + } else { + for (String index : indices) { + if (index == null) { + validationException = addValidationError("index is null", validationException); + break; + } + } + } + return validationException; + } + + @Override + public String[] indices() { + return this.indices; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + @Override + public CloneSnapshotRequest indices(String... indices) { + this.indices = indices; + return this; + } + + /** + * @see CloneSnapshotRequestBuilder#setIndicesOptions + */ + public CloneSnapshotRequest indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + public String repository() { + return this.repository; + } + + public String target() { + return this.target; + } + + public String source() { + return this.source; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java new file mode 100644 index 0000000000000..5139633fc2d26 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.clone; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.Strings; + +public class CloneSnapshotRequestBuilder extends MasterNodeOperationRequestBuilder<CloneSnapshotRequest, AcknowledgedResponse, + CloneSnapshotRequestBuilder> { + + protected CloneSnapshotRequestBuilder(ElasticsearchClient client, ActionType<AcknowledgedResponse> action, + CloneSnapshotRequest request) { + super(client, action, request); + } + + public CloneSnapshotRequestBuilder(ElasticsearchClient client, ActionType<AcknowledgedResponse> action, + String repository, String source, String target) { + this(client, action, new CloneSnapshotRequest(repository, source, target, Strings.EMPTY_ARRAY)); + } + + /** + * Sets a list of indices that should be cloned from the source to the target snapshot + * <p> + * The list of indices supports multi-index syntax. For example: "+test*" ,"-test42" will clone all indices with + * prefix "test" except index "test42". + * + * @return this builder + */ + public CloneSnapshotRequestBuilder setIndices(String... indices) { + request.indices(indices); + return this; + } + + /** + * Specifies the indices options. Like what type of requested indices to ignore. For example indices that don't exist. + * + * @param indicesOptions the desired behaviour regarding indices options + * @return this request + */ + public CloneSnapshotRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) { + request.indicesOptions(indicesOptions); + return this; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java new file mode 100644 index 0000000000000..3e7b8a8630cd2 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.clone; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.snapshots.SnapshotsService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; + +/** + * Transport action for the clone snapshot operation. + */ +public final class TransportCloneSnapshotAction extends TransportMasterNodeAction<CloneSnapshotRequest, AcknowledgedResponse> { + + private final SnapshotsService snapshotsService; + + @Inject + public TransportCloneSnapshotAction(TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, SnapshotsService snapshotsService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(CloneSnapshotAction.NAME, transportService, clusterService, threadPool, actionFilters, CloneSnapshotRequest::new, + indexNameExpressionResolver); + this.snapshotsService = snapshotsService; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected AcknowledgedResponse read(StreamInput in) throws IOException { + return new AcknowledgedResponse(in); + } + + @Override + protected void masterOperation(CloneSnapshotRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) { + throw new UnsupportedOperationException("not implemented yet"); + } + + @Override + protected ClusterBlockException checkBlock(CloneSnapshotRequest request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java index cd9adabaf2759..4701c67987f9b 100644 --- a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java @@ -71,6 +71,8 @@ import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequestBuilder; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; +import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; @@ -506,7 +508,22 @@ public interface ClusterAdminClient extends ElasticsearchClient { CreateSnapshotRequestBuilder prepareCreateSnapshot(String repository, String name); /** - * Get snapshot. + * Clones a snapshot. + */ + CloneSnapshotRequestBuilder prepareCloneSnapshot(String repository, String source, String target); + + /** + * Clones a snapshot. + */ + ActionFuture<AcknowledgedResponse> cloneSnapshot(CloneSnapshotRequest request); + + /** + * Clones a snapshot. + */ + void cloneSnapshot(CloneSnapshotRequest request, ActionListener<AcknowledgedResponse> listener); + + /** + * Get snapshots. */ ActionFuture<GetSnapshotsResponse> getSnapshots(GetSnapshotsRequest request); diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 879c60d065166..12fae90ab950a 100644 --- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -94,6 +94,9 @@ import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequestBuilder; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; +import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotAction; +import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder; @@ -959,6 +962,21 @@ public CreateSnapshotRequestBuilder prepareCreateSnapshot(String repository, Str return new CreateSnapshotRequestBuilder(this, CreateSnapshotAction.INSTANCE, repository, name); } + @Override + public CloneSnapshotRequestBuilder prepareCloneSnapshot(String repository, String source, String target) { + return new CloneSnapshotRequestBuilder(this, CloneSnapshotAction.INSTANCE, repository, source, target); + } + + @Override + public ActionFuture<AcknowledgedResponse> cloneSnapshot(CloneSnapshotRequest request) { + return execute(CloneSnapshotAction.INSTANCE, request); + } + + @Override + public void cloneSnapshot(CloneSnapshotRequest request, ActionListener<AcknowledgedResponse> listener) { + execute(CloneSnapshotAction.INSTANCE, request, listener); + } + @Override public ActionFuture<GetSnapshotsResponse> getSnapshots(GetSnapshotsRequest request) { return execute(GetSnapshotsAction.INSTANCE, request); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCloneSnapshotAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCloneSnapshotAction.java new file mode 100644 index 0000000000000..f22f1d95aefbd --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCloneSnapshotAction.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.cluster; + +import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +/** + * Clones indices from one snapshot into another snapshot in the same repository + */ +public class RestCloneSnapshotAction extends BaseRestHandler { + + @Override + public List<Route> routes() { + return Collections.singletonList(new Route(PUT, "/_snapshot/{repository}/{snapshot}/_clone/{target_snapshot}")); + } + + @Override + public String getName() { + return "clone_snapshot_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final Map<String, Object> source = request.contentParser().map(); + final CloneSnapshotRequest cloneSnapshotRequest = new CloneSnapshotRequest( + request.param("repository"), request.param("snapshot"), request.param("target_snapshot"), + XContentMapValues.nodeStringArrayValue(source.getOrDefault("indices", Collections.emptyList()))); + cloneSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", cloneSnapshotRequest.masterNodeTimeout())); + cloneSnapshotRequest.indicesOptions(IndicesOptions.fromMap(source, cloneSnapshotRequest.indicesOptions())); + return channel -> client.admin().cluster().cloneSnapshot(cloneSnapshotRequest, new RestToXContentListener<>(channel)); + } +}