From 3e77c5291856ce1cfb708cf803855af906e3efe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Sun, 2 Aug 2020 13:30:14 +0200 Subject: [PATCH 01/19] Add repositories stats tracking API --- .../repositories/azure/AzureRepository.java | 18 +- .../gcs/GoogleCloudStorageRepository.java | 16 +- .../repositories/s3/S3Repository.java | 17 +- .../repositories/RepositoriesService.java | 41 +- .../RepositoriesStatsArchive.java | 70 ++++ .../repositories/Repository.java | 5 +- .../repositories/RepositoryInfo.java | 133 +++++++ .../repositories/RepositoryStats.java | 14 + .../repositories/RepositoryStatsSnapshot.java | 91 +++++ .../blobstore/BlobStoreRepository.java | 3 +- .../blobstore/MeteredBlobStoreRepository.java | 55 +++ .../RepositoriesServiceTests.java | 55 ++- .../RepositoriesStatsArchiveTests.java | 85 ++++ .../fixtures/azure-fixture/docker-compose.yml | 9 + test/fixtures/gcs-fixture/docker-compose.yml | 12 + test/fixtures/s3-fixture/docker-compose.yml | 15 + ...ESMockAPIBasedRepositoryIntegTestCase.java | 7 +- .../test/rest/ESRestTestCase.java | 53 +++ x-pack/plugin/repositories-stats/build.gradle | 46 +++ .../repositories-stats/qa/azure/build.gradle | 101 +++++ .../stats/azure/AzureRepositoriesStatsIT.java | 48 +++ .../plugin/repositories-stats/qa/build.gradle | 6 + .../repositories-stats/qa/gcs/build.gradle | 132 +++++++ .../stats/gcs/GCSRepositoriesStatsIT.java | 47 +++ .../repositories-stats/qa/s3/build.gradle | 77 ++++ .../stats/s3/S3RepositoriesStatsIT.java | 48 +++ .../stats/RepositoriesStatsPlugin.java | 54 +++ .../ClearRepositoriesStatsArchiveAction.java | 19 + ...arRepositoriesStatsArchiveNodeRequest.java | 20 + ...rRepositoriesStatsArchiveNodeResponse.java | 23 ++ .../ClearRepositoriesStatsArchiveRequest.java | 22 ++ ...ClearRepositoriesStatsArchiveResponse.java | 59 +++ .../action/RepositoriesNodeStatsRequest.java | 20 + .../action/RepositoriesNodeStatsResponse.java | 50 +++ .../stats/action/RepositoriesStatsAction.java | 19 + .../action/RepositoriesStatsRequest.java | 22 ++ .../action/RepositoriesStatsResponse.java | 53 +++ ...rtClearRepositoriesStatsArchiveAction.java | 77 ++++ .../TransportRepositoriesStatsAction.java | 76 ++++ ...stClearRepositoriesStatsArchiveAction.java | 40 ++ .../rest/RestGetRepositoriesStatsAction.java | 41 ++ ...tractRepositoriesStatsAPIRestTestCase.java | 367 ++++++++++++++++++ .../RepositoriesStatsResponseTests.java | 100 +++++ .../TransportRepositoryStatsAction.java | 6 +- ...stractSearchableSnapshotsRestTestCase.java | 41 -- 45 files changed, 2258 insertions(+), 55 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java create mode 100644 server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java create mode 100644 server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java create mode 100644 server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java create mode 100644 server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java create mode 100644 x-pack/plugin/repositories-stats/build.gradle create mode 100644 x-pack/plugin/repositories-stats/qa/azure/build.gradle create mode 100644 x-pack/plugin/repositories-stats/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/stats/azure/AzureRepositoriesStatsIT.java create mode 100644 x-pack/plugin/repositories-stats/qa/build.gradle create mode 100644 x-pack/plugin/repositories-stats/qa/gcs/build.gradle create mode 100644 x-pack/plugin/repositories-stats/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/stats/gcs/GCSRepositoriesStatsIT.java create mode 100644 x-pack/plugin/repositories-stats/qa/s3/build.gradle create mode 100644 x-pack/plugin/repositories-stats/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/stats/s3/S3RepositoriesStatsIT.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/RepositoriesStatsPlugin.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeRequest.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeResponse.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveResponse.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsRequest.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsResponse.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsAction.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsRequest.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponse.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java create mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestGetRepositoriesStatsAction.java create mode 100644 x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java create mode 100644 x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 024d52f40e9ba..90393db05b2fe 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -33,7 +33,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.indices.recovery.RecoverySettings; -import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import java.util.Locale; import java.util.function.Function; @@ -52,7 +52,7 @@ *
{@code compress}
If set to true metadata files will be stored compressed. Defaults to false.
* */ -public class AzureRepository extends BlobStoreRepository { +public class AzureRepository extends MeteredBlobStoreRepository { private static final Logger logger = LogManager.getLogger(AzureRepository.class); public static final String TYPE = "azure"; @@ -75,6 +75,7 @@ public static final class Repository { private final ByteSizeValue chunkSize; private final AzureStorageService storageService; private final boolean readonly; + private final String container; public AzureRepository( final RepositoryMetadata metadata, @@ -94,6 +95,7 @@ public AzureRepository( } else { this.readonly = locationMode == LocationMode.SECONDARY_ONLY; } + this.container = Repository.CONTAINER_SETTING.get(metadata.settings()); } private static BlobPath buildBasePath(RepositoryMetadata metadata) { @@ -134,4 +136,16 @@ protected ByteSizeValue chunkSize() { public boolean isReadOnly() { return readonly; } + + @Override + protected String location() { + BlobPath location = BlobPath.cleanPath(); + + location = location.add(container); + for (String path : basePath()) { + location = location.add(path); + } + + return location.buildAsString(); + } } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index a50e7b617b774..a65c0a0d54242 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -31,7 +31,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoryException; -import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import java.util.function.Function; @@ -39,7 +39,7 @@ import static org.elasticsearch.common.settings.Setting.byteSizeSetting; import static org.elasticsearch.common.settings.Setting.simpleString; -class GoogleCloudStorageRepository extends BlobStoreRepository { +class GoogleCloudStorageRepository extends MeteredBlobStoreRepository { private static final Logger logger = LogManager.getLogger(GoogleCloudStorageRepository.class); // package private for testing @@ -118,4 +118,16 @@ static T getSetting(Setting setting, RepositoryMetadata metadata) { } return value; } + + @Override + protected String location() { + BlobPath location = BlobPath.cleanPath(); + + location = location.add(bucket); + for (String path : basePath()) { + location = location.add(path); + } + + return location.buildAsString(); + } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 51032981a9eef..bb08bfad976eb 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -41,7 +41,7 @@ import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.ShardGenerations; -import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotsService; @@ -66,7 +66,7 @@ *
{@code compress}
If set to true metadata files will be stored compressed. Defaults to false.
* */ -class S3Repository extends BlobStoreRepository { +class S3Repository extends MeteredBlobStoreRepository { private static final Logger logger = LogManager.getLogger(S3Repository.class); static final String TYPE = "s3"; @@ -328,4 +328,17 @@ protected void doClose() { } super.doClose(); } + + @Override + protected String location() { + logger.info("LOCATION {}", bucket); + BlobPath location = BlobPath.cleanPath(); + + location = location.add(bucket); + for (String path : basePath()) { + location = location.add(path); + } + + return location.buildAsString(); + } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index a3bc51bec2e37..7dff66814d4b8 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -43,7 +43,9 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.threadpool.ThreadPool; @@ -56,6 +58,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Service responsible for maintaining and providing access to snapshot repositories on nodes. @@ -64,6 +69,9 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C private static final Logger logger = LogManager.getLogger(RepositoriesService.class); + public static final Setting REPOSITORIES_STATS_ARCHIVE_RETENTION_PERIOD = + Setting.positiveTimeSetting("repositories.stats.archive.retention_period", TimeValue.timeValueHours(2), Setting.Property.NodeScope); + private final Map typesRegistry; private final Map internalTypesRegistry; @@ -75,6 +83,7 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C private final Map internalRepositories = ConcurrentCollections.newConcurrentMap(); private volatile Map repositories = Collections.emptyMap(); + private final RepositoriesStatsArchive repositoriesStatsArchive; public RepositoriesService(Settings settings, ClusterService clusterService, TransportService transportService, Map typesRegistry, Map internalTypesRegistry, @@ -89,6 +98,7 @@ public RepositoriesService(Settings settings, ClusterService clusterService, Tra clusterService.addHighPriorityApplier(this); } this.verifyAction = new VerifyNodeRepositoryAction(transportService, clusterService, this); + this.repositoriesStatsArchive = new RepositoriesStatsArchive(REPOSITORIES_STATS_ARCHIVE_RETENTION_PERIOD.get(settings)); } /** @@ -123,7 +133,7 @@ public void registerRepository(final PutRepositoryRequest request, final ActionL // Trying to create the new repository on master to make sure it works try { - closeRepository(createRepository(newRepositoryMetadata, typesRegistry)); + closeRepository(createRepository(newRepositoryMetadata, typesRegistry), false); } catch (Exception e) { registrationListener.onFailure(e); return; @@ -397,6 +407,27 @@ public Repository repository(String repositoryName) { throw new RepositoryMissingException(repositoryName); } + public List repositoriesStats() { + List archivedRepoStats = repositoriesStatsArchive.getArchivedStats(); + List activeRepoStats = getRepositoryStatsForActiveRepositories(); + + List repositoriesStats = new ArrayList<>(archivedRepoStats); + repositoriesStats.addAll(activeRepoStats); + return repositoriesStats; + } + + private List getRepositoryStatsForActiveRepositories() { + return Stream.concat(repositories.values().stream(), internalRepositories.values().stream()) + .map(Repository::statsSnapshot) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + public void clearRepositoriesStatsArchive() { + repositoriesStatsArchive.clear(); + } + public void registerInternalRepository(String name, String type) { RepositoryMetadata metadata = new RepositoryMetadata(name, type, Settings.EMPTY); Repository repository = internalRepositories.computeIfAbsent(name, (n) -> { @@ -423,8 +454,16 @@ public void unregisterInternalRepository(String name) { /** Closes the given repository. */ private void closeRepository(Repository repository) { + closeRepository(repository, true); + } + + private void closeRepository(Repository repository, boolean archiveStats) { logger.debug("closing repository [{}][{}]", repository.getMetadata().type(), repository.getMetadata().name()); repository.close(); + if (archiveStats) { + Optional stats = repository.statsSnapshot(); + stats.ifPresent(repositoriesStatsArchive::archive); + } } /** diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java new file mode 100644 index 0000000000000..510f4d630382d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories; + +import org.elasticsearch.common.unit.TimeValue; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +public final class RepositoriesStatsArchive { + private final TimeValue retentionPeriod; + private final Deque archive = new ArrayDeque<>(); + + RepositoriesStatsArchive(TimeValue retentionPeriod) { + this.retentionPeriod = retentionPeriod; + } + + synchronized void archive(RepositoryStatsSnapshot repositoryStats) { + if (repositoryStats.getRepositoryInfo().isStopped() == false) { + repositoryStats = repositoryStats.withStoppedRepo(); + } + archive.add(repositoryStats); + evict(); + } + + synchronized List getArchivedStats() { + evict(); + return List.copyOf(archive); + } + + synchronized void clear() { + archive.clear(); + } + + private void evict() { + Instant retentionDeadline = getRetentionDeadline(); + RepositoryStatsSnapshot stats; + while ((stats = archive.peek()) != null && shouldEvict(stats, retentionDeadline)) { + archive.poll(); + } + } + + private boolean shouldEvict(RepositoryStatsSnapshot stats, Instant deadline) { + return stats.wasRepoStoppedBefore(deadline); + } + + private Instant getRetentionDeadline() { + return Instant.now().minus(Duration.ofMillis(retentionPeriod.getMillis())); + } +} diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index ec0fb5b561ad1..36ad304402720 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; @@ -154,8 +155,8 @@ void deleteSnapshots(Collection snapshotIds, long repositoryStateId, /** * Returns stats on the repository usage */ - default RepositoryStats stats() { - return RepositoryStats.EMPTY_STATS; + default Optional statsSnapshot() { + return Optional.empty(); } /** diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java new file mode 100644 index 0000000000000..260619996ac9d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; + +public final class RepositoryInfo implements Writeable, ToXContentFragment { + public final String ephemeralId; + public final String name; + public final String type; + public final String location; + public final Instant startedAt; + @Nullable + public final Instant stoppedAt; + + public RepositoryInfo(String ephemeralId, + String name, + String type, + String location, + Instant startedAt) { + this(ephemeralId, name, type, location, startedAt, null); + } + + public RepositoryInfo(String ephemeralId, + String name, + String type, + String location, + Instant startedAt, + @Nullable Instant stoppedAt) { + this.ephemeralId = ephemeralId; + this.name = name; + this.type = type; + this.location = location; + this.startedAt = startedAt; + if (stoppedAt != null && startedAt.isAfter(stoppedAt)) { + throw new IllegalArgumentException("createdAt must be before or equal to stoppedAt"); + } + this.stoppedAt = stoppedAt; + } + + public RepositoryInfo(StreamInput in) throws IOException { + this.ephemeralId = in.readString(); + this.name = in.readString(); + this.type = in.readString(); + this.location = in.readString(); + this.startedAt = in.readInstant(); + this.stoppedAt = in.readOptionalInstant(); + } + + public RepositoryInfo stopped() { + assert isStopped() == false : "The repository is already stopped"; + + return new RepositoryInfo(ephemeralId, name, type, location, startedAt, Instant.now()); + } + + public boolean wasStoppedBefore(Instant instant) { + if (stoppedAt == null) { + return false; + } + return stoppedAt.isBefore(instant); + } + + public boolean isStopped() { + return stoppedAt != null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(ephemeralId); + out.writeString(name); + out.writeString(type); + out.writeString(location); + out.writeInstant(startedAt); + out.writeOptionalInstant(stoppedAt); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("repository_name", name); + builder.field("repository_type", type); + builder.field("repository_location", location); + builder.field("repository_ephemeral_id", ephemeralId); + builder.field("repository_started_at", startedAt.toEpochMilli()); + if (stoppedAt != null) { + builder.field("repository_stopped_at", stoppedAt.toEpochMilli()); + } + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RepositoryInfo that = (RepositoryInfo) o; + return Objects.equals(ephemeralId, that.ephemeralId) && + Objects.equals(name, that.name) && + Objects.equals(type, that.type) && + Objects.equals(location, that.location) && + Objects.equals(startedAt, that.startedAt) && + Objects.equals(stoppedAt, that.stoppedAt); + } + + @Override + public int hashCode() { + return Objects.hash(ephemeralId, name, type, location, startedAt, stoppedAt); + } +} diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java index 50a8b46630179..53ed90f828b2c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; public class RepositoryStats implements Writeable { @@ -55,4 +56,17 @@ public RepositoryStats merge(RepositoryStats otherStats) { public void writeTo(StreamOutput out) throws IOException { out.writeMap(requestCounts, StreamOutput::writeString, StreamOutput::writeLong); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RepositoryStats that = (RepositoryStats) o; + return Objects.equals(requestCounts, that.requestCounts); + } + + @Override + public int hashCode() { + return Objects.hash(requestCounts); + } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java new file mode 100644 index 0000000000000..e5b9062340ee6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; + +public final class RepositoryStatsSnapshot implements Writeable, ToXContent { + private final RepositoryInfo repositoryInfo; + private final RepositoryStats repositoryStats; + + public RepositoryStatsSnapshot(RepositoryInfo repositoryInfo, + RepositoryStats repositoryStats) { + this.repositoryInfo = repositoryInfo; + this.repositoryStats = repositoryStats; + } + + public RepositoryStatsSnapshot(StreamInput in) throws IOException { + this.repositoryInfo = new RepositoryInfo(in); + this.repositoryStats = new RepositoryStats(in); + } + + public RepositoryInfo getRepositoryInfo() { + return repositoryInfo; + } + + public RepositoryStats getRepositoryStats() { + return repositoryStats; + } + + public boolean wasRepoStoppedBefore(Instant instant) { + return repositoryInfo.wasStoppedBefore(instant); + } + + public RepositoryStatsSnapshot withStoppedRepo() { + return new RepositoryStatsSnapshot(repositoryInfo.stopped(), repositoryStats); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + repositoryInfo.writeTo(out); + repositoryStats.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + repositoryInfo.toXContent(builder, params); + builder.field("request_counts", repositoryStats.requestCounts); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RepositoryStatsSnapshot that = (RepositoryStatsSnapshot) o; + return Objects.equals(repositoryInfo, that.repositoryInfo) && + Objects.equals(repositoryStats, that.repositoryStats); + } + + @Override + public int hashCode() { + return Objects.hash(repositoryInfo, repositoryStats); + } +} diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 5b85057306550..c626a181831c0 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -533,8 +533,7 @@ public RepositoryMetadata getMetadata() { return metadata; } - @Override - public RepositoryStats stats() { + protected RepositoryStats stats() { final BlobStore store = blobStore.get(); if (store == null) { return RepositoryStats.EMPTY_STATS; diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java new file mode 100644 index 0000000000000..f01205afb7dec --- /dev/null +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.blobstore; + +import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.repositories.RepositoryInfo; +import org.elasticsearch.repositories.RepositoryStatsSnapshot; + +import java.time.Instant; +import java.util.Optional; + +public abstract class MeteredBlobStoreRepository extends BlobStoreRepository { + private final String ephemeralId; + private final Instant startedAt; + + public MeteredBlobStoreRepository(RepositoryMetadata metadata, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + RecoverySettings recoverySettings, + BlobPath basePath) { + super(metadata, namedXContentRegistry, clusterService, recoverySettings, basePath); + this.ephemeralId = UUIDs.randomBase64UUID(); + this.startedAt = Instant.now(); + } + + @Override + public Optional statsSnapshot() { + RepositoryInfo repositoryInfo = new RepositoryInfo(ephemeralId, metadata.name(), metadata.type(), location(), startedAt); + return Optional.of(new RepositoryStatsSnapshot(repositoryInfo, stats())); + } + + protected abstract String location(); +} diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 6a7738c1c32e4..a65da23c4c46b 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -47,12 +47,16 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; +import java.time.Instant; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; +import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; public class RepositoriesServiceTests extends ESTestCase { @@ -67,8 +71,10 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null, Collections.emptySet()); + Map typesRegistry = Map.of(TestRepository.TYPE, TestRepository::new, + SecondTestRepository.TYPE, SecondTestRepository::new); repositoriesService = new RepositoriesService(Settings.EMPTY, mock(ClusterService.class), - transportService, Collections.emptyMap(), Collections.singletonMap(TestRepository.TYPE, TestRepository::new), threadPool); + transportService, Collections.emptyMap(), typesRegistry, threadPool); repositoriesService.start(); } @@ -114,6 +120,28 @@ public void testRegisterRejectsInvalidRepositoryNames() { } } + public void testRepositoriesStatsCanHaveTheSameNameAndDifferentTypeOverTime() { + String repoName = "name"; + expectThrows(RepositoryMissingException.class, () -> repositoriesService.repository(repoName)); + repositoriesService.registerInternalRepository(repoName, TestRepository.TYPE); + assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); + + repositoriesService.unregisterInternalRepository(repoName); + assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); + + repositoriesService.registerInternalRepository(repoName, SecondTestRepository.TYPE); + + List repositoriesStats = repositoriesService.repositoriesStats(); + assertThat(repositoriesStats.size(), equalTo(2)); + RepositoryStatsSnapshot repositoryStats = repositoriesStats.get(0); + assertThat(repositoryStats.getRepositoryInfo().type, equalTo(TestRepository.TYPE)); + assertThat(repositoryStats.getRepositoryStats().requestCounts, equalTo(Map.of("TEST-REPO", 10L))); + + RepositoryStatsSnapshot repositoryStats2 = repositoriesStats.get(1); + assertThat(repositoryStats2.getRepositoryInfo().type, equalTo(SecondTestRepository.TYPE)); + assertThat(repositoryStats2.getRepositoryStats().requestCounts, equalTo(Map.of("SECOND-TEST", 20L))); + } + private void assertThrowsOnRegister(String repoName) { PutRepositoryRequest request = new PutRepositoryRequest(repoName); expectThrows(RepositoryException.class, () -> repositoriesService.registerRepository(request, null)); @@ -122,6 +150,7 @@ private void assertThrowsOnRegister(String repoName) { private static class TestRepository implements Repository { private static final String TYPE = "internal"; + private static final String LOCATION = "location"; private boolean isClosed; private boolean isStarted; @@ -256,5 +285,29 @@ public void stop() { public void close() { isClosed = true; } + + @Override + public Optional statsSnapshot() { + RepositoryInfo repositoryInfo = + new RepositoryInfo(UUIDs.randomBase64UUID(), metadata.name(), metadata.type(), LOCATION, Instant.now()); + return Optional.of(new RepositoryStatsSnapshot(repositoryInfo, getRepositoryStats())); + } + + protected RepositoryStats getRepositoryStats() { + return new RepositoryStats(Map.of("TEST-REPO", 10L)); + } + } + + private static class SecondTestRepository extends TestRepository { + private static final String TYPE = "second-internal"; + + private SecondTestRepository(RepositoryMetadata metadata) { + super(metadata); + } + + @Override + protected RepositoryStats getRepositoryStats() { + return new RepositoryStats(Map.of("SECOND-TEST", 20L)); + } } } diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java new file mode 100644 index 0000000000000..5fcf055a9936e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class RepositoriesStatsArchiveTests extends ESTestCase { + public void testStatsAreEvictedOnceTheyAreOlderThanRetentionPeriod() { + int retentionTimeInHours = randomIntBetween(1, 4); + RepositoriesStatsArchive repositoriesStatsArchive = + new RepositoriesStatsArchive(TimeValue.timeValueHours(retentionTimeInHours)); + + int statsOlderThanRetentionPeriodCount = randomInt(10); + for (int i = 0; i < statsOlderThanRetentionPeriodCount; i++) { + RepositoryInfo repositoryInfo = + createRepositoryInfo(retentionTimeInHours + 2, retentionTimeInHours + 1); + + RepositoryStatsSnapshot repoStats = + new RepositoryStatsSnapshot(repositoryInfo, RepositoryStats.EMPTY_STATS); + + repositoriesStatsArchive.archive(repoStats); + } + + int statsToBeRetainedCount = randomInt(10); + for (int i = 0; i < statsToBeRetainedCount; i++) { + RepositoryInfo repositoryInfo = + createRepositoryInfo(0); + + RepositoryStatsSnapshot repoStats = + new RepositoryStatsSnapshot(repositoryInfo, new RepositoryStats(Map.of("GET", 10L))); + repositoriesStatsArchive.archive(repoStats); + } + + List archivedStats = repositoriesStatsArchive.getArchivedStats(); + assertThat(archivedStats.size(), equalTo(statsToBeRetainedCount)); + for (RepositoryStatsSnapshot repositoryStatsSnapshot : archivedStats) { + assertThat(repositoryStatsSnapshot.getRepositoryStats().requestCounts, equalTo(Map.of("GET", 10L))); + } + } + + private RepositoryInfo createRepositoryInfo(int hoursSinceRepoStarted) { + return new RepositoryInfo(UUIDs.randomBase64UUID(), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Instant.now().minus(Duration.ofHours(hoursSinceRepoStarted)), + null); + } + + private RepositoryInfo createRepositoryInfo(int hoursSinceRepoStarted, int hoursSinceRepoStopped) { + assert hoursSinceRepoStarted > hoursSinceRepoStopped; + return new RepositoryInfo(UUIDs.randomBase64UUID(), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Instant.now().minus(Duration.ofHours(hoursSinceRepoStarted)), + Instant.now().minus(Duration.ofHours(hoursSinceRepoStopped))); + } +} diff --git a/test/fixtures/azure-fixture/docker-compose.yml b/test/fixtures/azure-fixture/docker-compose.yml index 61ea9d28a560a..782ba8127dc89 100644 --- a/test/fixtures/azure-fixture/docker-compose.yml +++ b/test/fixtures/azure-fixture/docker-compose.yml @@ -17,3 +17,12 @@ services: - ./testfixtures_shared/shared:/fixture/shared ports: - "8091" + + azure-fixture-repositories-stats: + build: + context: . + dockerfile: Dockerfile + volumes: + - ./testfixtures_shared/shared:/fixture/shared + ports: + - "8091" diff --git a/test/fixtures/gcs-fixture/docker-compose.yml b/test/fixtures/gcs-fixture/docker-compose.yml index a53c4366df6dc..2c8cba84a01d4 100644 --- a/test/fixtures/gcs-fixture/docker-compose.yml +++ b/test/fixtures/gcs-fixture/docker-compose.yml @@ -36,3 +36,15 @@ services: - ./testfixtures_shared/shared:/fixture/shared ports: - "80" + gcs-fixture-repositories-stats: + build: + context: . + args: + port: 80 + bucket: "bucket" + token: "o/oauth2/token" + dockerfile: Dockerfile + volumes: + - ./testfixtures_shared/shared:/fixture/shared + ports: + - "80" diff --git a/test/fixtures/s3-fixture/docker-compose.yml b/test/fixtures/s3-fixture/docker-compose.yml index 1d06334eddbd3..70d494b5990d4 100644 --- a/test/fixtures/s3-fixture/docker-compose.yml +++ b/test/fixtures/s3-fixture/docker-compose.yml @@ -30,6 +30,21 @@ services: ports: - "80" + s3-fixture-repositories-stats: + build: + context: . + args: + fixtureClass: fixture.s3.S3HttpFixture + port: 80 + bucket: "bucket" + basePath: "base_path" + accessKey: "access_key" + dockerfile: Dockerfile + volumes: + - ./testfixtures_shared/shared:/fixture/shared + ports: + - "80" + s3-fixture-with-session-token: build: context: . diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java index 03a69db208530..69ee1fdd3c8ef 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java @@ -39,6 +39,7 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryStats; +import org.elasticsearch.repositories.RepositoryStatsSnapshot; import org.elasticsearch.test.BackgroundIndexer; import org.junit.After; import org.junit.AfterClass; @@ -53,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -211,7 +213,10 @@ public void testRequestStats() throws Exception { } }) .filter(Objects::nonNull) - .map(Repository::stats) + .map(Repository::statsSnapshot) + .filter(Optional::isPresent) + .map(Optional::get) + .map(RepositoryStatsSnapshot::getRepositoryStats) .reduce(RepositoryStats::merge) .get(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 80ea7f439e66c..756d9dff7d478 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -23,6 +23,8 @@ import org.apache.http.HttpHost; import org.apache.http.HttpStatus; import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.apache.http.message.BasicHeader; import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.http.ssl.SSLContexts; @@ -31,6 +33,7 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; +import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RequestOptions.Builder; @@ -1183,6 +1186,10 @@ protected static Map getAlias(final String index, final String a protected static Map getAsMap(final String endpoint) throws IOException { Response response = client().performRequest(new Request("GET", endpoint)); + return responseAsMap(response); + } + + protected static Map responseAsMap(Response response) throws IOException { XContentType entityContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue()); Map responseEntity = XContentHelper.convertToMap(entityContentType.xContent(), response.getEntity().getContent(), false); @@ -1190,6 +1197,52 @@ protected static Map getAsMap(final String endpoint) throws IOEx return responseEntity; } + protected static void registerRepository(String repository, String type, boolean verify, Settings settings) throws IOException { + final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + repository); + request.addParameter("verify", Boolean.toString(verify)); + request.setJsonEntity(Strings.toString(new PutRepositoryRequest(repository).type(type).settings(settings))); + + final Response response = client().performRequest(request); + assertAcked("Failed to create repository [" + repository + "] of type [" + type + "]: " + response, response); + } + + protected static void createSnapshot(String repository, String snapshot, boolean waitForCompletion) throws IOException { + final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); + request.addParameter("wait_for_completion", Boolean.toString(waitForCompletion)); + + final Response response = client().performRequest(request); + assertThat( + "Failed to create snapshot [" + snapshot + "] in repository [" + repository + "]: " + response, + response.getStatusLine().getStatusCode(), + equalTo(RestStatus.OK.getStatus()) + ); + } + + protected static void restoreSnapshot(String repository, String snapshot, boolean waitForCompletion) throws IOException { + final Request request = new Request(HttpPost.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot + "/_restore"); + request.addParameter("wait_for_completion", Boolean.toString(waitForCompletion)); + + final Response response = client().performRequest(request); + assertThat( + "Failed to restore snapshot [" + snapshot + "] from repository [" + repository + "]: " + response, + response.getStatusLine().getStatusCode(), + equalTo(RestStatus.OK.getStatus()) + ); + } + + @SuppressWarnings("unchecked") + private static void assertAcked(String message, Response response) throws IOException { + final int responseStatusCode = response.getStatusLine().getStatusCode(); + assertThat( + message + ": expecting response code [200] but got [" + responseStatusCode + ']', + responseStatusCode, + equalTo(RestStatus.OK.getStatus()) + ); + final Map responseAsMap = responseAsMap(response); + Boolean acknowledged = (Boolean) XContentMapValues.extractValue(responseAsMap, "acknowledged"); + assertThat(message + ": response is not acknowledged", acknowledged, equalTo(Boolean.TRUE)); + } + /** * Is this template one that is automatically created by xpack? */ diff --git a/x-pack/plugin/repositories-stats/build.gradle b/x-pack/plugin/repositories-stats/build.gradle new file mode 100644 index 0000000000000..2784fe02dd5c1 --- /dev/null +++ b/x-pack/plugin/repositories-stats/build.gradle @@ -0,0 +1,46 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' +esplugin { + name 'repositories-stats' + description 'Repositories stats API' + classname 'org.elasticsearch.xpack.repositories.stats.RepositoriesStatsPlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-repositories-stats' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +// xpack modules are installed in real clusters as the meta plugin, so +// installing them as individual plugins for integ tests doesn't make sense, +// so we disable integ tests +integTest.enabled = false + +// add all sub-projects of the qa sub-project +gradle.projectsEvaluated { + project.subprojects + .find { it.path == project.path + ":qa" } + .subprojects + .findAll { it.path.startsWith(project.path + ":qa") } + .each { check.dependsOn it.check } +} + +configurations { + testArtifacts.extendsFrom testRuntime + testArtifacts.extendsFrom testImplementation +} + +task testJar(type: Jar) { + appendix 'test' + from sourceSets.test.output +} + +artifacts { + testArtifacts testJar +} + +test { +} diff --git a/x-pack/plugin/repositories-stats/qa/azure/build.gradle b/x-pack/plugin/repositories-stats/qa/azure/build.gradle new file mode 100644 index 0000000000000..76eeab2c5c2ed --- /dev/null +++ b/x-pack/plugin/repositories-stats/qa/azure/build.gradle @@ -0,0 +1,101 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import org.elasticsearch.gradle.info.BuildParams +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.rest-resources' + +final Project fixture = project(':test:fixtures:azure-fixture') +final Project repositoryPlugin = project(':plugins:repository-azure') + +dependencies { + testImplementation project(path: xpackModule('repositories-stats'), configuration: 'testArtifacts') + testImplementation repositoryPlugin +} + +restResources { + restApi { + includeCore 'indices', 'bulk', 'snapshot', 'nodes', '_common' + includeXpack 'repositories-stats' + } +} + +boolean useFixture = false +String azureAccount = System.getenv("azure_storage_account") +String azureKey = System.getenv("azure_storage_key") +String azureContainer = System.getenv("azure_storage_container") +String azureBasePath = System.getenv("azure_storage_base_path") +String azureSasToken = System.getenv("azure_storage_sas_token") + +if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) { + azureAccount = 'azure_integration_test_account' + azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64 + azureContainer = 'container' + azureBasePath = '' + azureSasToken = '' + useFixture = true + +} + +if (useFixture) { + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture(fixture.path, 'azure-fixture-repositories-stats') +} + +integTest { + dependsOn repositoryPlugin.bundlePlugin + runner { + systemProperty 'test.azure.container', azureContainer + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repositories_stats_tests_" + BuildParams.testSeed + } +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + plugin repositoryPlugin.bundlePlugin.archiveFile + + keystore 'azure.client.repositories_stats.account', azureAccount + if (azureKey != null && azureKey.isEmpty() == false) { + keystore 'azure.client.repositories_stats.key', azureKey + } + if (azureSasToken != null && azureSasToken.isEmpty() == false) { + keystore 'azure.client.repositories_stats.sas_token', azureSasToken + } + + if (useFixture) { + def fixtureAddress = { fixtureName -> + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.8091" + assert ephemeralPort > 0 + '127.0.0.1:' + ephemeralPort + } + setting 'azure.client.repositories_stats.endpoint_suffix', + { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${-> fixtureAddress('azure-fixture-repositories-stats')}" }, IGNORE_VALUE + + } else { + println "Using an external service to test " + project.name + } +} + +task azureThirdPartyTest { + dependsOn integTest +} + diff --git a/x-pack/plugin/repositories-stats/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/stats/azure/AzureRepositoriesStatsIT.java b/x-pack/plugin/repositories-stats/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/stats/azure/AzureRepositoriesStatsIT.java new file mode 100644 index 0000000000000..aed4d77ea100b --- /dev/null +++ b/x-pack/plugin/repositories-stats/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/stats/azure/AzureRepositoriesStatsIT.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.repositories.stats.azure; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.repositories.stats.AbstractRepositoriesStatsAPIRestTestCase; + +import java.util.List; + +public class AzureRepositoriesStatsIT extends AbstractRepositoriesStatsAPIRestTestCase { + + @Override + protected String repositoryType() { + return "azure"; + } + + @Override + protected String repositoryLocation() { + return getProperty("test.azure.container") + "/" + getProperty("test.azure.base_path") + "/"; + } + + @Override + protected Settings repositorySettings() { + final String container = getProperty("test.azure.container"); + + final String basePath = getProperty("test.azure.base_path"); + + return Settings.builder().put("client", "repositories_stats").put("container", container).put("base_path", basePath).build(); + } + + @Override + protected Settings updatedRepositorySettings() { + return Settings.builder().put(repositorySettings()).put("azure.client.repositories_stats.max_retries", 5).build(); + } + + @Override + protected List readCounterKeys() { + return List.of("GET", "HEAD", "LIST"); + } + + @Override + protected List writeCounterKeys() { + return List.of("PUT"); + } +} diff --git a/x-pack/plugin/repositories-stats/qa/build.gradle b/x-pack/plugin/repositories-stats/qa/build.gradle new file mode 100644 index 0000000000000..53a1915bb2a07 --- /dev/null +++ b/x-pack/plugin/repositories-stats/qa/build.gradle @@ -0,0 +1,6 @@ +apply plugin: 'elasticsearch.build' +test.enabled = false + +dependencies { + api project(':test:framework') +} diff --git a/x-pack/plugin/repositories-stats/qa/gcs/build.gradle b/x-pack/plugin/repositories-stats/qa/gcs/build.gradle new file mode 100644 index 0000000000000..7e2907863dae6 --- /dev/null +++ b/x-pack/plugin/repositories-stats/qa/gcs/build.gradle @@ -0,0 +1,132 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.MavenFilteringHack + +import java.nio.file.Files +import java.security.KeyPair +import java.security.KeyPairGenerator + +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.rest-resources' + +final Project fixture = project(':test:fixtures:gcs-fixture') +final Project repositoryPlugin = project(':plugins:repository-gcs') + +dependencies { + testImplementation project(path: xpackModule('repositories-stats'), configuration: 'testArtifacts') + testImplementation repositoryPlugin +} + +restResources { + restApi { + includeCore 'indices', 'bulk', 'snapshot', 'nodes', '_common' + includeXpack 'repositories-stats' + } +} + +boolean useFixture = false + +String gcsServiceAccount = System.getenv("google_storage_service_account") +String gcsBucket = System.getenv("google_storage_bucket") +String gcsBasePath = System.getenv("google_storage_base_path") + +File serviceAccountFile = null +if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { + serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json') + gcsBucket = 'bucket' + gcsBasePath = 'integration_test' + useFixture = true +} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) { + throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present") +} else { + serviceAccountFile = new File(gcsServiceAccount) +} + +/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/ +task createServiceAccountFile() { + doLast { + KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA") + keyPairGenerator.initialize(1024) + KeyPair keyPair = keyPairGenerator.generateKeyPair() + String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded()) + + serviceAccountFile.parentFile.mkdirs() + serviceAccountFile.setText("{\n" + + ' "type": "service_account",\n' + + ' "project_id": "integration_test",\n' + + ' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' + + ' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' + + ' "client_email": "integration_test@appspot.gserviceaccount.com",\n' + + ' "client_id": "123456789101112130594"\n' + + '}', 'UTF-8') + } +} + +def fixtureAddress = { f -> + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${f}.tcp.80" + assert ephemeralPort > 0 + 'http://127.0.0.1:' + ephemeralPort +} + +Map expansions = [ + 'bucket' : gcsBucket, + 'base_path': gcsBasePath + "_integration_tests" +] + +processTestResources { + inputs.properties(expansions) + MavenFilteringHack.filter(it, expansions) +} + +if (useFixture) { + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture(fixture.path, 'gcs-fixture-repositories-stats') +} + +integTest { + dependsOn repositoryPlugin.bundlePlugin + runner { + systemProperty 'test.gcs.bucket', gcsBucket + nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repositories_stats" + BuildParams.testSeed + } +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + plugin repositoryPlugin.bundlePlugin.archiveFile + + keystore 'gcs.client.repositories_stats.credentials_file', serviceAccountFile, IGNORE_VALUE + if (useFixture) { + tasks.integTest.dependsOn createServiceAccountFile + /* Use a closure on the string to delay evaluation until tests are executed */ + setting 'gcs.client.repositories_stats.endpoint', { "${-> fixtureAddress('gcs-fixture-repositories-stats')}" }, IGNORE_VALUE + setting 'gcs.client.repositories_stats.token_uri', { "${-> fixtureAddress('gcs-fixture-repositories-stats')}/o/oauth2/token" }, IGNORE_VALUE + } else { + println "Using an external service to test " + project.name + } +} + +task gcsThirdPartyTest { + dependsOn integTest +} diff --git a/x-pack/plugin/repositories-stats/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/stats/gcs/GCSRepositoriesStatsIT.java b/x-pack/plugin/repositories-stats/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/stats/gcs/GCSRepositoriesStatsIT.java new file mode 100644 index 0000000000000..f96238804a47d --- /dev/null +++ b/x-pack/plugin/repositories-stats/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/stats/gcs/GCSRepositoriesStatsIT.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.repositories.stats.gcs; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.repositories.stats.AbstractRepositoriesStatsAPIRestTestCase; + +import java.util.List; + +public class GCSRepositoriesStatsIT extends AbstractRepositoriesStatsAPIRestTestCase { + + @Override + protected String repositoryType() { + return "gcs"; + } + + @Override + protected String repositoryLocation() { + return getProperty("test.gcs.bucket") + "/" + getProperty("test.gcs.base_path") + "/"; + } + + @Override + protected Settings repositorySettings() { + final String bucket = getProperty("test.gcs.bucket"); + final String basePath = getProperty("test.gcs.base_path"); + + return Settings.builder().put("client", "repositories_stats").put("bucket", bucket).put("base_path", basePath).build(); + } + + @Override + protected Settings updatedRepositorySettings() { + return Settings.builder().put(repositorySettings()).put("gcs.client.repositories_stats.application_name", "updated").build(); + } + + @Override + protected List readCounterKeys() { + return List.of("GET", "LIST"); + } + + @Override + protected List writeCounterKeys() { + return List.of("POST"); + } +} diff --git a/x-pack/plugin/repositories-stats/qa/s3/build.gradle b/x-pack/plugin/repositories-stats/qa/s3/build.gradle new file mode 100644 index 0000000000000..a75d7550d09cb --- /dev/null +++ b/x-pack/plugin/repositories-stats/qa/s3/build.gradle @@ -0,0 +1,77 @@ +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE +import org.elasticsearch.gradle.info.BuildParams + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.rest-resources' + +final Project fixture = project(':test:fixtures:s3-fixture') +final Project repositoryPlugin = project(':plugins:repository-s3') + +dependencies { + testImplementation project(path: xpackModule('repositories-stats'), configuration: 'testArtifacts') + testImplementation repositoryPlugin +} + +restResources { + restApi { + includeCore 'indices', 'bulk', 'snapshot', 'nodes', '_common' + includeXpack 'repositories-stats-api' + } +} + +boolean useFixture = false +String s3AccessKey = System.getenv("amazon_s3_access_key") +String s3SecretKey = System.getenv("amazon_s3_secret_key") +String s3Bucket = System.getenv("amazon_s3_bucket") +String s3BasePath = System.getenv("amazon_s3_base_path") + +if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) { + s3AccessKey = 'access_key' + s3SecretKey = 'secret_key' + s3Bucket = 'bucket' + s3BasePath = null + useFixture = true + +} else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) { + throw new IllegalArgumentException("not all options specified to run against external S3 service are present") +} + +if (useFixture) { + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture(fixture.path, 's3-fixture-repositories-stats') +} + +integTest { + dependsOn repositoryPlugin.bundlePlugin + runner { + systemProperty 'test.s3.bucket', s3Bucket + nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_repositories_metering" + BuildParams.testSeed : 'base_path' + } +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + plugin repositoryPlugin.bundlePlugin.archiveFile + + keystore 's3.client.repositories_metering.access_key', s3AccessKey + keystore 's3.client.repositories_metering.secret_key', s3SecretKey + + if (useFixture) { + def fixtureAddress = { fixtureName -> + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80" + assert ephemeralPort > 0 + '127.0.0.1:' + ephemeralPort + } + setting 's3.client.repositories_metering.protocol', 'http' + setting 's3.client.repositories_metering.endpoint', { "${-> fixtureAddress('s3-fixture-repositories-stats')}" }, IGNORE_VALUE + + } else { + println "Using an external service to test " + project.name + } +} + +task s3ThirdPartyTest { + dependsOn integTest +} diff --git a/x-pack/plugin/repositories-stats/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/stats/s3/S3RepositoriesStatsIT.java b/x-pack/plugin/repositories-stats/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/stats/s3/S3RepositoriesStatsIT.java new file mode 100644 index 0000000000000..d88bcb896daf1 --- /dev/null +++ b/x-pack/plugin/repositories-stats/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/stats/s3/S3RepositoriesStatsIT.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.repositories.stats.s3; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.repositories.stats.AbstractRepositoriesStatsAPIRestTestCase; + +import java.util.List; + +public class S3RepositoriesStatsIT extends AbstractRepositoriesStatsAPIRestTestCase { + + @Override + protected String repositoryType() { + return "s3"; + } + + @Override + protected String repositoryLocation() { + return getProperty("test.s3.bucket") + "/" + getProperty("test.s3.base_path") + "/"; + } + + @Override + protected Settings repositorySettings() { + final String bucket = getProperty("test.s3.bucket"); + final String basePath = getProperty("test.s3.base_path"); + + return Settings.builder().put("client", "repositories_metering").put("bucket", bucket).put("base_path", basePath).build(); + } + + @Override + protected Settings updatedRepositorySettings() { + Settings settings = repositorySettings(); + return Settings.builder().put(settings).put("s3.client.max_retries", 4).build(); + } + + @Override + protected List readCounterKeys() { + return List.of("GET", "LIST"); + } + + @Override + protected List writeCounterKeys() { + return List.of("PUT"); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/RepositoriesStatsPlugin.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/RepositoriesStatsPlugin.java new file mode 100644 index 0000000000000..239c01e5e2173 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/RepositoriesStatsPlugin.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.xpack.repositories.stats.action.ClearRepositoriesStatsArchiveAction; +import org.elasticsearch.xpack.repositories.stats.action.RepositoriesStatsAction; +import org.elasticsearch.xpack.repositories.stats.action.TransportClearRepositoriesStatsArchiveAction; +import org.elasticsearch.xpack.repositories.stats.action.TransportRepositoriesStatsAction; +import org.elasticsearch.xpack.repositories.stats.rest.RestClearRepositoriesStatsArchiveAction; +import org.elasticsearch.xpack.repositories.stats.rest.RestGetRepositoriesStatsAction; + +import java.util.List; +import java.util.function.Supplier; + +public final class RepositoriesStatsPlugin extends Plugin implements ActionPlugin { + + @Override + public List> getActions() { + return List.of( + new ActionHandler<>(RepositoriesStatsAction.INSTANCE, TransportRepositoriesStatsAction.class), + new ActionHandler<>(ClearRepositoriesStatsArchiveAction.INSTANCE, TransportClearRepositoriesStatsArchiveAction.class) + ); + } + + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + + return List.of(new RestGetRepositoriesStatsAction(), new RestClearRepositoriesStatsArchiveAction()); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java new file mode 100644 index 0000000000000..317e37c034275 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.ActionType; + +public final class ClearRepositoriesStatsArchiveAction extends ActionType { + public static final ClearRepositoriesStatsArchiveAction INSTANCE = new ClearRepositoriesStatsArchiveAction(); + + static final String NAME = "cluster:monitor/xpack/repositories_stats/clear_repositories_stats_archive"; + + ClearRepositoriesStatsArchiveAction() { + super(NAME, ClearRepositoriesStatsArchiveResponse::new); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeRequest.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeRequest.java new file mode 100644 index 0000000000000..5f9e8847cf33e --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeRequest.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.transport.TransportRequest; + +import java.io.IOException; + +final class ClearRepositoriesStatsArchiveNodeRequest extends TransportRequest { + ClearRepositoriesStatsArchiveNodeRequest() {} + + ClearRepositoriesStatsArchiveNodeRequest(StreamInput in) throws IOException { + super(in); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeResponse.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeResponse.java new file mode 100644 index 0000000000000..47c93cabcb2d0 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeResponse.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; + +final class ClearRepositoriesStatsArchiveNodeResponse extends BaseNodeResponse { + ClearRepositoriesStatsArchiveNodeResponse(StreamInput in) throws IOException { + super(in); + } + + ClearRepositoriesStatsArchiveNodeResponse(DiscoveryNode node) { + super(node); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java new file mode 100644 index 0000000000000..ec0fbbc515448 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; + +public final class ClearRepositoriesStatsArchiveRequest extends BaseNodesRequest { + public ClearRepositoriesStatsArchiveRequest(StreamInput in) throws IOException { + super(in); + } + + public ClearRepositoriesStatsArchiveRequest(String... nodesIds) { + super(nodesIds); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveResponse.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveResponse.java new file mode 100644 index 0000000000000..b65c4ab5f1198 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveResponse.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; + +public class ClearRepositoriesStatsArchiveResponse extends BaseNodesResponse + implements + ToXContentObject { + + public ClearRepositoriesStatsArchiveResponse(StreamInput in) throws IOException { + super(in); + } + + public ClearRepositoriesStatsArchiveResponse( + ClusterName clusterName, + List nodes, + List failures + ) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(ClearRepositoriesStatsArchiveNodeResponse::new); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (failures().isEmpty() == false) { + builder.startArray("failures"); + for (FailedNodeException failure : failures()) { + builder.startObject(); + failure.toXContent(builder, params); + builder.endObject(); + } + builder.endArray(); + } + return builder; + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsRequest.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsRequest.java new file mode 100644 index 0000000000000..e943efe807c33 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsRequest.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.transport.TransportRequest; + +import java.io.IOException; + +final class RepositoriesNodeStatsRequest extends TransportRequest { + RepositoriesNodeStatsRequest() {} + + RepositoriesNodeStatsRequest(StreamInput in) throws IOException { + super(in); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsResponse.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsResponse.java new file mode 100644 index 0000000000000..fcda075d572f3 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsResponse.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.repositories.RepositoryStatsSnapshot; + +import java.io.IOException; +import java.util.List; + +public final class RepositoriesNodeStatsResponse extends BaseNodeResponse implements ToXContentFragment { + + final List repositoryStatsSnapshots; + + public RepositoriesNodeStatsResponse(DiscoveryNode node, List repositoryStatsSnapshots) { + super(node); + this.repositoryStatsSnapshots = repositoryStatsSnapshots; + } + + public RepositoriesNodeStatsResponse(StreamInput in) throws IOException { + super(in); + this.repositoryStatsSnapshots = in.readList(RepositoryStatsSnapshot::new); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startArray(getNode().getId()); + for (RepositoryStatsSnapshot repositoryStats : repositoryStatsSnapshots) { + repositoryStats.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeList(repositoryStatsSnapshots); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsAction.java new file mode 100644 index 0000000000000..a554a26c252fa --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsAction.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.ActionType; + +public final class RepositoriesStatsAction extends ActionType { + public static final RepositoriesStatsAction INSTANCE = new RepositoriesStatsAction(); + + static final String NAME = "cluster:monitor/xpack/repositories_stats/get_stats"; + + RepositoriesStatsAction() { + super(NAME, RepositoriesStatsResponse::new); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsRequest.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsRequest.java new file mode 100644 index 0000000000000..562281ffa6324 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsRequest.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; + +public final class RepositoriesStatsRequest extends BaseNodesRequest { + public RepositoriesStatsRequest(StreamInput in) throws IOException { + super(in); + } + + public RepositoriesStatsRequest(String... nodesIds) { + super(nodesIds); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponse.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponse.java new file mode 100644 index 0000000000000..ecdcedf534377 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponse.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; + +public final class RepositoriesStatsResponse extends BaseNodesResponse implements ToXContentFragment { + + public RepositoriesStatsResponse(StreamInput in) throws IOException { + super(in); + } + + public RepositoriesStatsResponse( + ClusterName clusterName, + List nodes, + List failures + ) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(RepositoriesNodeStatsResponse::new); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("nodes"); + for (RepositoriesNodeStatsResponse nodeStats : getNodes()) { + nodeStats.toXContent(builder, params); + } + builder.endObject(); + return builder; + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java new file mode 100644 index 0000000000000..73ae0e6d31b7f --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +public final class TransportClearRepositoriesStatsArchiveAction extends TransportNodesAction< + ClearRepositoriesStatsArchiveRequest, + ClearRepositoriesStatsArchiveResponse, + ClearRepositoriesStatsArchiveNodeRequest, + ClearRepositoriesStatsArchiveNodeResponse> { + + private final RepositoriesService repositoriesService; + + @Inject + public TransportClearRepositoriesStatsArchiveAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + RepositoriesService repositoriesService + ) { + super( + ClearRepositoriesStatsArchiveAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + ClearRepositoriesStatsArchiveRequest::new, + ClearRepositoriesStatsArchiveNodeRequest::new, + ThreadPool.Names.SAME, + ClearRepositoriesStatsArchiveNodeResponse.class + ); + this.repositoriesService = repositoriesService; + } + + @Override + protected ClearRepositoriesStatsArchiveResponse newResponse( + ClearRepositoriesStatsArchiveRequest request, + List nodesResponses, + List failures + ) { + return new ClearRepositoriesStatsArchiveResponse(clusterService.getClusterName(), nodesResponses, failures); + } + + @Override + protected ClearRepositoriesStatsArchiveNodeRequest newNodeRequest(ClearRepositoriesStatsArchiveRequest request) { + return new ClearRepositoriesStatsArchiveNodeRequest(); + } + + @Override + protected ClearRepositoriesStatsArchiveNodeResponse newNodeResponse(StreamInput in) throws IOException { + return new ClearRepositoriesStatsArchiveNodeResponse(in); + } + + @Override + protected ClearRepositoriesStatsArchiveNodeResponse nodeOperation(ClearRepositoriesStatsArchiveNodeRequest request, Task task) { + repositoriesService.clearRepositoriesStatsArchive(); + return new ClearRepositoriesStatsArchiveNodeResponse(clusterService.localNode()); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java new file mode 100644 index 0000000000000..ffa64b8723da4 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +public final class TransportRepositoriesStatsAction extends TransportNodesAction< + RepositoriesStatsRequest, + RepositoriesStatsResponse, + RepositoriesNodeStatsRequest, + RepositoriesNodeStatsResponse> { + + private final RepositoriesService repositoriesService; + + @Inject + public TransportRepositoriesStatsAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + RepositoriesService repositoriesService + ) { + super( + RepositoriesStatsAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + RepositoriesStatsRequest::new, + RepositoriesNodeStatsRequest::new, + ThreadPool.Names.SAME, + RepositoriesNodeStatsResponse.class + ); + this.repositoriesService = repositoriesService; + } + + @Override + protected RepositoriesStatsResponse newResponse( + RepositoriesStatsRequest request, + List repositoriesNodeStatsResponses, + List failures + ) { + return new RepositoriesStatsResponse(clusterService.getClusterName(), repositoriesNodeStatsResponses, failures); + } + + @Override + protected RepositoriesNodeStatsRequest newNodeRequest(RepositoriesStatsRequest request) { + return new RepositoriesNodeStatsRequest(); + } + + @Override + protected RepositoriesNodeStatsResponse newNodeResponse(StreamInput in) throws IOException { + return new RepositoriesNodeStatsResponse(in); + } + + @Override + protected RepositoriesNodeStatsResponse nodeOperation(RepositoriesNodeStatsRequest request, Task task) { + return new RepositoriesNodeStatsResponse(clusterService.localNode(), repositoriesService.repositoriesStats()); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java new file mode 100644 index 0000000000000..cf48e8fc41398 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.rest; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.repositories.stats.action.ClearRepositoriesStatsArchiveAction; +import org.elasticsearch.xpack.repositories.stats.action.ClearRepositoriesStatsArchiveRequest; + +import java.util.List; + +public class RestClearRepositoriesStatsArchiveAction extends BaseRestHandler { + @Override + public String getName() { + return "clear_repositories_stats_action"; + } + + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.DELETE, "/_nodes/{nodeId}/_repositories_stats")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); + ClearRepositoriesStatsArchiveRequest clearArchivesRequest = new ClearRepositoriesStatsArchiveRequest(nodesIds); + return channel -> client.execute( + ClearRepositoriesStatsArchiveAction.INSTANCE, + clearArchivesRequest, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestGetRepositoriesStatsAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestGetRepositoriesStatsAction.java new file mode 100644 index 0000000000000..0c28a2ef9d608 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestGetRepositoriesStatsAction.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.rest; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.xpack.repositories.stats.action.RepositoriesStatsRequest; +import org.elasticsearch.xpack.repositories.stats.action.RepositoriesStatsAction; + +import java.util.List; + +public final class RestGetRepositoriesStatsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_repositories_stats_action"; + } + + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.GET, "/_nodes/{nodeId}/_repositories_stats")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); + RepositoriesStatsRequest repositoriesStatsRequest = new RepositoriesStatsRequest(nodesIds); + return channel -> client.execute( + RepositoriesStatsAction.INSTANCE, + repositoriesStatsRequest, + new RestActions.NodesResponseRestListener<>(channel) + ); + } +} diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java new file mode 100644 index 0000000000000..eac8bd47780f7 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java @@ -0,0 +1,367 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats; + +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.repositories.RepositoryInfo; +import org.elasticsearch.repositories.RepositoryStats; +import org.elasticsearch.repositories.RepositoryStatsSnapshot; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; + +public abstract class AbstractRepositoriesStatsAPIRestTestCase extends ESRestTestCase { + protected abstract String repositoryType(); + + protected abstract String repositoryLocation(); + + protected abstract Settings repositorySettings(); + + /** + * New settings to force a new repository creation + */ + protected abstract Settings updatedRepositorySettings(); + + protected abstract List readCounterKeys(); + + protected abstract List writeCounterKeys(); + + @Before + public void clear() throws Exception { + clearRepositoriesStats(); + } + + @After + public void clearStats() throws Exception { + clearRepositoriesStats(); + } + + public void testStatsAreTracked() throws Exception { + snapshotAndRestoreIndex((repository, index) -> { + List repoStats = getRepositoriesStats(); + assertThat(repoStats.size(), equalTo(1)); + + RepositoryStatsSnapshot repositoryStats = repoStats.get(0); + assertRepositoryStatsBelongToRepository(repositoryStats, repository); + assertRequestCountersAccountedForReads(repositoryStats); + assertRequestCountersAccountedForWrites(repositoryStats); + }); + } + + public void testStatsAreUpdatedAfterRepositoryOperations() throws Exception { + String snapshot = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + snapshotAndRestoreIndex(snapshot, (repository, index) -> { + List repoStatsBeforeRestore = getRepositoriesStats(); + assertThat(repoStatsBeforeRestore.size(), equalTo(1)); + + RepositoryStatsSnapshot repositoryStatsBeforeRestore = repoStatsBeforeRestore.get(0); + Map requestCountsBeforeRestore = repositoryStatsBeforeRestore.getRepositoryStats().requestCounts; + assertRepositoryStatsBelongToRepository(repositoryStatsBeforeRestore, repository); + assertRequestCountersAccountedForReads(repositoryStatsBeforeRestore); + assertRequestCountersAccountedForWrites(repositoryStatsBeforeRestore); + + deleteIndex(index); + + restoreSnapshot(repository, snapshot, true); + + List updatedRepoStats = getRepositoriesStats(); + assertThat(updatedRepoStats.size(), equalTo(1)); + RepositoryStatsSnapshot repoStatsAfterRestore = updatedRepoStats.get(0); + Map requestCountsAfterRestore = repoStatsAfterRestore.getRepositoryStats().requestCounts; + + for (String readCounterKey : readCounterKeys()) { + assertThat( + requestCountsAfterRestore.get(readCounterKey), + greaterThanOrEqualTo(requestCountsBeforeRestore.get(readCounterKey)) + ); + } + }); + } + + public void testClearRepositoriesStats() throws Exception { + snapshotAndRestoreIndex((repository, index) -> { + deleteRepository(repository); + + assertThat(getRepositoriesStats().size(), equalTo(1)); + + clearRepositoriesStats(); + + assertThat(getRepositoriesStats().size(), equalTo(0)); + }); + } + + public void testRegisterMultipleRepositoriesAndGetStats() throws Exception { + List repositoryNames = List.of("repo-a", "repo-b", "repo-c"); + for (String repositoryName : repositoryNames) { + registerRepository(repositoryName, repositoryType(), false, repositorySettings()); + } + + List repositoriesStats = getRepositoriesStats(); + Map> repositoryStatsByName = repositoriesStats.stream() + .collect(Collectors.groupingBy(r -> r.getRepositoryInfo().name)); + + for (String repositoryName : repositoryNames) { + List repositoryStats = repositoryStatsByName.get(repositoryName); + assertThat(repositoryStats, is(notNullValue())); + assertThat(repositoryStats.size(), equalTo(1)); + + RepositoryStatsSnapshot stats = repositoryStats.get(0); + assertRepositoryStatsBelongToRepository(stats, repositoryName); + assertAllRequestCountsAreZero(stats); + } + } + + public void testStatsAreArchivedAfterRepositoryDeletion() throws Exception { + snapshotAndRestoreIndex((repository, index) -> { + List repositoriesStats = getRepositoriesStats(); + assertThat(repositoriesStats.size(), equalTo(1)); + RepositoryStatsSnapshot statsBeforeRepoDeletion = repositoriesStats.get(0); + assertRepositoryStatsBelongToRepository(statsBeforeRepoDeletion, repository); + + deleteRepository(repository); + + List repoStatsAfterDeletion = getRepositoriesStats(); + assertThat(repoStatsAfterDeletion.size(), equalTo(1)); + RepositoryStatsSnapshot statsAfterRepoDeletion = repoStatsAfterDeletion.get(0); + assertStatsAreEqualsIgnoringStoppedAt(statsBeforeRepoDeletion, statsAfterRepoDeletion); + }); + } + + public void testStatsAreStoredIntoANewCounterInstanceAfterRepoConfigUpdate() throws Exception { + final String snapshot = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + snapshotAndRestoreIndex(snapshot, (repository, index) -> { + List repositoriesStatsBeforeUpdate = getRepositoriesStats(); + assertThat(repositoriesStatsBeforeUpdate.size(), equalTo(1)); + assertRepositoryStatsBelongToRepository(repositoriesStatsBeforeUpdate.get(0), repository); + assertRequestCountersAccountedForReads(repositoriesStatsBeforeUpdate.get(0)); + assertRequestCountersAccountedForWrites(repositoriesStatsBeforeUpdate.get(0)); + + // Update repository + registerRepository(repository, repositoryType(), false, updatedRepositorySettings()); + + List repositoriesStatsAfterUpdate = getRepositoriesStats(); + + assertThat(repositoriesStatsAfterUpdate.size(), equalTo(2)); + assertStatsAreEqualsIgnoringStoppedAt(repositoriesStatsBeforeUpdate.get(0), repositoriesStatsAfterUpdate.get(0)); + + // The counters for the new repository instance are zero + assertAllRequestCountsAreZero(repositoriesStatsAfterUpdate.get(1)); + + deleteIndex(index); + + restoreSnapshot(repository, snapshot, true); + + List repoStatsAfterRestore = getRepositoriesStats(); + + assertThat(repoStatsAfterRestore.size(), equalTo(2)); + assertStatsAreEqualsIgnoringStoppedAt(repositoriesStatsAfterUpdate.get(0), repoStatsAfterRestore.get(0)); + + assertRequestCountersAccountedForReads(repoStatsAfterRestore.get(1)); + }); + } + + public void testDeleteThenAddRepositoryWithTheSameName() throws Exception { + snapshotAndRestoreIndex((repository, index) -> { + List repoStatsBeforeDeletion = getRepositoriesStats(); + assertThat(repoStatsBeforeDeletion.size(), equalTo(1)); + + deleteRepository(repository); + + List repoStatsAfterDeletion = getRepositoriesStats(); + assertThat(repoStatsAfterDeletion.size(), equalTo(1)); + assertStatsAreEqualsIgnoringStoppedAt(repoStatsBeforeDeletion.get(0), repoStatsAfterDeletion.get(0)); + + registerRepository(repository, repositoryType(), false, repositorySettings()); + + List repositoriesStatsAfterRegisteringTheSameRepo = getRepositoriesStats(); + assertThat(repositoriesStatsAfterRegisteringTheSameRepo.size(), equalTo(2)); + assertStatsAreEqualsIgnoringStoppedAt(repoStatsBeforeDeletion.get(0), repositoriesStatsAfterRegisteringTheSameRepo.get(0)); + assertAllRequestCountsAreZero(repositoriesStatsAfterRegisteringTheSameRepo.get(1)); + }); + } + + private void snapshotAndRestoreIndex(CheckedBiConsumer biConsumer) throws Exception { + final String snapshot = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + snapshotAndRestoreIndex(snapshot, biConsumer); + } + + private void snapshotAndRestoreIndex(String snapshot, CheckedBiConsumer biConsumer) throws Exception { + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String repository = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final int numberOfShards = randomIntBetween(1, 5); + + final String repositoryType = repositoryType(); + final Settings repositorySettings = repositorySettings(); + + registerRepository(repository, repositoryType, true, repositorySettings); + + createIndex( + indexName, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build() + ); + ensureGreen(indexName); + + final int numDocs = randomIntBetween(1, 500); + final StringBuilder bulkBody = new StringBuilder(); + for (int i = 0; i < numDocs; i++) { + bulkBody.append("{\"index\":{\"_id\":\"").append(i).append("\"}}\n"); + bulkBody.append("{\"field\":").append(i).append(",\"text\":\"Document number ").append(i).append("\"}\n"); + } + + final Request documents = new Request(HttpPost.METHOD_NAME, '/' + indexName + "/_bulk"); + documents.addParameter("refresh", Boolean.TRUE.toString()); + documents.setJsonEntity(bulkBody.toString()); + assertOK(client().performRequest(documents)); + + createSnapshot(repository, snapshot, true); + + deleteIndex(indexName); + + restoreSnapshot(repository, snapshot, true); + + biConsumer.accept(repository, indexName); + } + + private void assertRequestCountersAccountedForReads(RepositoryStatsSnapshot statsSnapshot) { + RepositoryStats repositoryStats = statsSnapshot.getRepositoryStats(); + Map requestCounts = repositoryStats.requestCounts; + for (String readCounterKey : readCounterKeys()) { + assertThat(requestCounts.get(readCounterKey), is(notNullValue())); + assertThat(requestCounts.get(readCounterKey), is(greaterThan(0L))); + } + } + + private void assertRequestCountersAccountedForWrites(RepositoryStatsSnapshot statsSnapshot) { + RepositoryStats repositoryStats = statsSnapshot.getRepositoryStats(); + Map requestCounts = repositoryStats.requestCounts; + for (String writeCounterKey : writeCounterKeys()) { + assertThat(requestCounts.get(writeCounterKey), is(notNullValue())); + assertThat(requestCounts.get(writeCounterKey), is(greaterThan(0L))); + } + } + + private void assertStatsAreEqualsIgnoringStoppedAt(RepositoryStatsSnapshot stats, RepositoryStatsSnapshot otherStats) { + assertRepositoryInfoIsEqualIgnoringStoppedAt(stats.getRepositoryInfo(), otherStats.getRepositoryInfo()); + assertThat(stats.getRepositoryStats(), equalTo(otherStats.getRepositoryStats())); + } + + private void assertRepositoryInfoIsEqualIgnoringStoppedAt(RepositoryInfo repositoryInfo, RepositoryInfo otherRepositoryInfo) { + assertThat(repositoryInfo.ephemeralId, equalTo(otherRepositoryInfo.ephemeralId)); + assertThat(repositoryInfo.name, equalTo(otherRepositoryInfo.name)); + assertThat(repositoryInfo.type, equalTo(otherRepositoryInfo.type)); + assertThat(repositoryInfo.location, equalTo(otherRepositoryInfo.location)); + assertThat(repositoryInfo.startedAt, equalTo(otherRepositoryInfo.startedAt)); + } + + private void assertRepositoryStatsBelongToRepository(RepositoryStatsSnapshot stats, String repositoryName) { + RepositoryInfo repositoryInfo = stats.getRepositoryInfo(); + assertThat(repositoryInfo.name, equalTo(repositoryName)); + assertThat(repositoryInfo.type, equalTo(repositoryType())); + assertThat(repositoryInfo.location, equalTo(repositoryLocation())); + } + + private void assertAllRequestCountsAreZero(RepositoryStatsSnapshot statsSnapshot) { + RepositoryStats stats = statsSnapshot.getRepositoryStats(); + for (long requestCount : stats.requestCounts.values()) { + assertThat(requestCount, equalTo(0)); + } + } + + private List getRepositoriesStats() throws IOException { + Map response = getAsMap("/_nodes/_all/_repositories_stats"); + Map>> nodesRepoStats = extractValue(response, "nodes"); + assertThat(response.size(), greaterThan(0)); + List repositoriesStats = new ArrayList<>(); + for (String nodeId : getNodeIds()) { + List> nodeStats = nodesRepoStats.get(nodeId); + assertThat(nodeStats, is(notNullValue())); + + for (Map nodeStatSnapshot : nodeStats) { + RepositoryInfo repositoryInfo = parseRepositoryInfo(nodeStatSnapshot); + Map intRequestCounters = extractValue(nodeStatSnapshot, "request_counts"); + Map requestCounters = intRequestCounters.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().longValue())); + RepositoryStats repositoryStats = new RepositoryStats(requestCounters); + RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot(repositoryInfo, repositoryStats); + repositoriesStats.add(statsSnapshot); + } + } + return repositoriesStats; + } + + private RepositoryInfo parseRepositoryInfo(Map nodeStatSnapshot) { + String id = extractValue(nodeStatSnapshot, "repository_ephemeral_id"); + String name = extractValue(nodeStatSnapshot, "repository_name"); + String type = extractValue(nodeStatSnapshot, "repository_type"); + String location = extractValue(nodeStatSnapshot, "repository_location"); + Long startedAt = extractValue(nodeStatSnapshot, "repository_started_at"); + Long stoppedAt = extractValue(nodeStatSnapshot, "repository_stopped_at"); + return new RepositoryInfo( + id, + name, + type, + location, + Instant.ofEpochMilli(startedAt), + stoppedAt != null ? Instant.ofEpochMilli(stoppedAt) : null + ); + } + + private Set getNodeIds() throws IOException { + Map nodes = extractValue(getAsMap("_nodes/"), "nodes"); + return nodes.keySet(); + } + + private void clearRepositoriesStats() throws IOException { + final Request request = new Request(HttpDelete.METHOD_NAME, "/_nodes/_all/_repositories_stats"); + final Response response = client().performRequest(request); + assertThat( + "Failed to clear repositories stats: " + response, + response.getStatusLine().getStatusCode(), + equalTo(RestStatus.OK.getStatus()) + ); + } + + @SuppressWarnings("unchecked") + protected static T extractValue(Map map, String path) { + return (T) XContentMapValues.extractValue(path, map); + } + + protected String getProperty(String propertyName) { + final String property = System.getProperty(propertyName); + assertThat(property, not(blankOrNullString())); + return property; + } +} diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java new file mode 100644 index 0000000000000..41020f1900c05 --- /dev/null +++ b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.stats.action; + +import org.elasticsearch.Version; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.repositories.RepositoryInfo; +import org.elasticsearch.repositories.RepositoryStats; +import org.elasticsearch.repositories.RepositoryStatsSnapshot; +import org.elasticsearch.test.ESTestCase; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class RepositoriesStatsResponseTests extends ESTestCase { + public void testSerializationRoundtrip() throws Exception { + final RepositoriesStatsResponse repositoriesStatsResponse = createResponse(); + final RepositoriesStatsResponse deserializedResponse = copyWriteable( + repositoriesStatsResponse, + writableRegistry(), + RepositoriesStatsResponse::new, + Version.CURRENT + ); + assertResponsesAreEqual(repositoriesStatsResponse, deserializedResponse); + } + + private void assertResponsesAreEqual(RepositoriesStatsResponse response, RepositoriesStatsResponse otherResponse) { + List nodeResponses = response.getNodes(); + List otherNodeResponses = otherResponse.getNodes(); + assertThat(nodeResponses.size(), equalTo(otherNodeResponses.size())); + for (int i = 0; i < nodeResponses.size(); i++) { + RepositoriesNodeStatsResponse nodeResponse = nodeResponses.get(i); + RepositoriesNodeStatsResponse otherNodeResponse = otherNodeResponses.get(i); + assertThat(nodeResponse.repositoryStatsSnapshots, equalTo(otherNodeResponse.repositoryStatsSnapshots)); + } + + List failures = response.failures(); + List otherFailures = otherResponse.failures(); + assertThat(failures.size(), equalTo(otherFailures.size())); + for (int i = 0; i < failures.size(); i++) { + FailedNodeException failure = failures.get(i); + FailedNodeException otherFailure = otherFailures.get(i); + assertThat(failure.nodeId(), equalTo(otherFailure.nodeId())); + assertThat(failure.getMessage(), equalTo(otherFailure.getMessage())); + } + } + + private RepositoriesStatsResponse createResponse() { + ClusterName clusterName = new ClusterName("test"); + int nodes = randomIntBetween(1, 10); + List nodeResponses = new ArrayList<>(nodes); + for (int i = 0; i < nodes; i++) { + DiscoveryNode node = new DiscoveryNode("nodeId" + i, buildNewFakeTransportAddress(), Version.CURRENT); + int numberOfRepos = randomInt(10); + List nodeRepoStats = new ArrayList<>(numberOfRepos); + + for (int j = 0; j < numberOfRepos; j++) { + String repoId = randomAlphaOfLength(10); + String repoName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + String repoType = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + String repoLocation = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + Instant startedAt = Instant.now().minus(Duration.ofMinutes(randomLongBetween(1, 30))); + Instant stoppedAt = randomBoolean() ? Instant.now() : null; + RepositoryInfo repositoryInfo = new RepositoryInfo(repoId, repoName, repoType, repoLocation, startedAt, stoppedAt); + RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot( + repositoryInfo, + new RepositoryStats(Map.of("GET", randomLongBetween(0, 2000))) + ); + nodeRepoStats.add(statsSnapshot); + } + + nodeResponses.add(new RepositoriesNodeStatsResponse(node, nodeRepoStats)); + } + + int numberOfFailures = randomInt(20); + List failures = new ArrayList<>(numberOfFailures); + for (int i = nodes; i < numberOfFailures + nodes; i++) { + FailedNodeException failedNodeException = new FailedNodeException( + "nodeId" + i, + "error", + randomBoolean() ? new RuntimeException("boom") : null + ); + failures.add(failedNodeException); + } + + return new RepositoriesStatsResponse(clusterName, nodeResponses, failures); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportRepositoryStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportRepositoryStatsAction.java index 2c1300820a4a2..660b780c2a114 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportRepositoryStatsAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportRepositoryStatsAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryStats; +import org.elasticsearch.repositories.RepositoryStatsSnapshot; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -84,6 +85,9 @@ protected RepositoryStatsNodeResponse nodeOperation(RepositoryStatsNodeRequest r return new RepositoryStatsNodeResponse(clusterService.localNode(), RepositoryStats.EMPTY_STATS); } final Repository repository = repositoriesService.repository(request.getRepository()); - return new RepositoryStatsNodeResponse(clusterService.localNode(), repository.stats()); + RepositoryStats repositoryStats = repository.statsSnapshot() + .map(RepositoryStatsSnapshot::getRepositoryStats) + .orElse(RepositoryStats.EMPTY_STATS); + return new RepositoryStatsNodeResponse(clusterService.localNode(), repositoryStats); } } diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java index 00ac39c0773fe..0dd84a588c00c 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java @@ -8,20 +8,14 @@ import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.query.QueryBuilder; @@ -31,7 +25,6 @@ import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; -import java.io.InputStream; import java.util.List; import java.util.Locale; import java.util.Map; @@ -39,7 +32,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.notNullValue; public abstract class AbstractSearchableSnapshotsRestTestCase extends ESRestTestCase { @@ -250,26 +242,6 @@ public void assertSearchResults(String indexName, int numDocs, Boolean ignoreThr } } - protected static void registerRepository(String repository, String type, boolean verify, Settings settings) throws IOException { - final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + repository); - request.setJsonEntity(Strings.toString(new PutRepositoryRequest(repository).type(type).verify(verify).settings(settings))); - - final Response response = client().performRequest(request); - assertAcked("Failed to create repository [" + repository + "] of type [" + type + "]: " + response, response); - } - - protected static void createSnapshot(String repository, String snapshot, boolean waitForCompletion) throws IOException { - final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); - request.addParameter("wait_for_completion", Boolean.toString(waitForCompletion)); - - final Response response = client().performRequest(request); - assertThat( - "Failed to create snapshot [" + snapshot + "] in repository [" + repository + "]: " + response, - response.getStatusLine().getStatusCode(), - equalTo(RestStatus.OK.getStatus()) - ); - } - protected static void deleteSnapshot(String repository, String snapshot, boolean ignoreMissing) throws IOException { final Request request = new Request(HttpDelete.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); try { @@ -406,19 +378,6 @@ protected static Map indexSettings(String index) throws IOExcept return extractValue(responseAsMap(response), index + ".settings"); } - protected static Map responseAsMap(Response response) throws IOException { - final XContentType xContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue()); - assertThat("Unknown XContentType", xContentType, notNullValue()); - - BytesReference bytesReference = Streams.readFully(response.getEntity().getContent()); - - try (InputStream responseBody = bytesReference.streamInput()) { - return XContentHelper.convertToMap(xContentType.xContent(), responseBody, true); - } catch (Exception e) { - throw new IOException(bytesReference.utf8ToString(), e); - } - } - @SuppressWarnings("unchecked") protected static T extractValue(Map map, String path) { return (T) XContentMapValues.extractValue(path, map); From 528da9a3a87816f7da98d9a3accc49021c4f98f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Mon, 3 Aug 2020 12:29:19 +0200 Subject: [PATCH 02/19] Add docs --- .../clear-repositories-stats-archive.asciidoc | 31 ++++ .../apis/get-repositories-stats.asciidoc | 142 ++++++++++++++++++ .../apis/repositories-stats.asciidoc | 12 ++ 3 files changed, 185 insertions(+) create mode 100644 docs/reference/repositories-stats/apis/clear-repositories-stats-archive.asciidoc create mode 100644 docs/reference/repositories-stats/apis/get-repositories-stats.asciidoc create mode 100644 docs/reference/repositories-stats/apis/repositories-stats.asciidoc diff --git a/docs/reference/repositories-stats/apis/clear-repositories-stats-archive.asciidoc b/docs/reference/repositories-stats/apis/clear-repositories-stats-archive.asciidoc new file mode 100644 index 0000000000000..d7e25bfbfb730 --- /dev/null +++ b/docs/reference/repositories-stats/apis/clear-repositories-stats-archive.asciidoc @@ -0,0 +1,31 @@ +[[clear-repositories-stats-archive-api]] +=== Clear repositories statistics archive API +++++ +Clear repositories statistics archive API +++++ + +Removes the archived repositories statistics in the cluster. + +[[clear-repositories-stats-archive-api-request]] +==== {api-request-title} + +`DELETE /_nodes//_repositories_stats` + +[[clear-repositories-stats-archive-api-desc]] +==== {api-description-title} + +You can clear the archived repositories utilization statistics using this API. + +All the nodes selective options are explained <>. + +[[clear-repositories-stats-archive-api-path-params]] +==== {api-path-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] + +[[clear-repositories-stats-archive-api-query-params]] +==== {api-query-parms-title} + +[role="child_attributes"] +[[clear-repositories-stats-archive-api-response-body]] +==== {api-response-body-title} diff --git a/docs/reference/repositories-stats/apis/get-repositories-stats.asciidoc b/docs/reference/repositories-stats/apis/get-repositories-stats.asciidoc new file mode 100644 index 0000000000000..9ee5b8eef5953 --- /dev/null +++ b/docs/reference/repositories-stats/apis/get-repositories-stats.asciidoc @@ -0,0 +1,142 @@ +[[get-repositories-stats-api]] +=== Repositories statistics API +++++ +Repositories statistics +++++ + +Returns cluster repositories utilization statistics. + +[[get-repositories-stats-api-request]] +==== {api-request-title} + +`GET /_nodes//_repositories_stats` + +[[get-repositories-stats-api-desc]] +==== {api-description-title} + +You can use the cluster repositories statistics API to retrieve repository utilization statistics in a cluster. + + +All the nodes selective options are explained <>. + +[[get-repositories-stats-api-path-params]] +==== {api-path-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] + +[[get-repositories-stats-api-query-params]] +==== {api-query-parms-title} + +[role="child_attributes"] +[[get-repositories-stats-api-response-body]] +==== {api-response-body-title} + +`_nodes`:: +(object) +Contains statistics about the number of nodes selected by the request. ++ +.Properties of `_nodes` +[%collapsible%open] +==== +`total`:: +(integer) +Total number of nodes selected by the request. + +`successful`:: +(integer) +Number of nodes that responded successfully to the request. + +`failed`:: +(integer) +Number of nodes that rejected the request or failed to respond. If this value +is not `0`, a reason for the rejection or failure is included in the response. +==== + +`cluster_name`:: +(string) +Name of the cluster. Based on the <> setting. + +`nodes`:: +(object) +Contains repository utilization statistics for the nodes selected by the request. ++ +.Properties of `nodes` +[%collapsible%open] +==== +``:: +(array) +An array of repository utilization statistics for the node. ++ +.Properties of objects in `node_id` +[%collapsible%open] +===== +`repository_name`:: +(string) +Repository name. + +`repository_type`:: +(string) +Repository type. + +`repository_location`:: +(string) +Represents an unique location within the repository, +i.e. a `bucket` + `base_path`. + +`repository_ephemeral_id`:: +(string) +An identifier that changes every time the repository is updated. + +`repository_started_at`:: +(long) +Time the repository was created or updated. Recorded in milliseconds +since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. + +`repository_stopped_at`:: +(long) +Time the repository was deleted or updated. Recorded in milliseconds +since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. + +`request_counts`:: +(object) +An object with the number of request performed against the repository +grouped by request type. ++ +.Properties of `request_counts` for repository type `Azure` +[%collapsible%open] +====== +`HEAD`:: +(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-properties[Get Blob Properties] requests. +`GET`:: +(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob[Get Blob] requests. +`PUT`:: +(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob[Put Blob] requests. +====== ++ +.Properties of `request_counts` for repository type `GCP` +[%collapsible%open] +====== +`GET`:: +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/get[GET] requests. +`LIST`:: +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/list[LIST] requests. +`PUT`:: +(long) Number of https://cloud.google.com/storage/docs/performing-resumable-uploads[PUT] requests. +`POST`:: +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/insert[POST] requests. +====== ++ +.Properties of `request_counts` for repository type `S3` +[%collapsible%open] +====== +`GET`:: +(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html[GET] requests. +`LIST`:: +(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html[LIST] requests. +`PUT`:: +(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html[PUT] requests. +`POST`:: +(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html[MultiPart] requests. +====== +===== +==== diff --git a/docs/reference/repositories-stats/apis/repositories-stats.asciidoc b/docs/reference/repositories-stats/apis/repositories-stats.asciidoc new file mode 100644 index 0000000000000..2ebb0d9da2c89 --- /dev/null +++ b/docs/reference/repositories-stats/apis/repositories-stats.asciidoc @@ -0,0 +1,12 @@ +[[repositories-stats-apis]] +== Repositories utilization statistics APIs + +experimental[] + +You can use the following APIs to retrieve statistics about the repositories utilization. + +* <> +* <> + +include::get-repositories-stats.asciidoc[] +include::clear-repositories-stats-archive.asciidoc[] From f71fabbf109a52d46c87f76d4609b6b1f720f065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Mon, 3 Aug 2020 13:28:09 +0200 Subject: [PATCH 03/19] Fix build scripts --- x-pack/plugin/repositories-stats/qa/azure/build.gradle | 6 ++---- x-pack/plugin/repositories-stats/qa/gcs/build.gradle | 6 ++---- x-pack/plugin/repositories-stats/qa/s3/build.gradle | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/repositories-stats/qa/azure/build.gradle b/x-pack/plugin/repositories-stats/qa/azure/build.gradle index 76eeab2c5c2ed..c9d53346e66c8 100644 --- a/x-pack/plugin/repositories-stats/qa/azure/build.gradle +++ b/x-pack/plugin/repositories-stats/qa/azure/build.gradle @@ -62,10 +62,8 @@ if (useFixture) { integTest { dependsOn repositoryPlugin.bundlePlugin - runner { - systemProperty 'test.azure.container', azureContainer - nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repositories_stats_tests_" + BuildParams.testSeed - } + systemProperty 'test.azure.container', azureContainer + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repositories_stats_tests_" + BuildParams.testSeed } testClusters.integTest { diff --git a/x-pack/plugin/repositories-stats/qa/gcs/build.gradle b/x-pack/plugin/repositories-stats/qa/gcs/build.gradle index 7e2907863dae6..0ce6accdd12d1 100644 --- a/x-pack/plugin/repositories-stats/qa/gcs/build.gradle +++ b/x-pack/plugin/repositories-stats/qa/gcs/build.gradle @@ -106,10 +106,8 @@ if (useFixture) { integTest { dependsOn repositoryPlugin.bundlePlugin - runner { - systemProperty 'test.gcs.bucket', gcsBucket - nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repositories_stats" + BuildParams.testSeed - } + systemProperty 'test.gcs.bucket', gcsBucket + nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repositories_stats" + BuildParams.testSeed } testClusters.integTest { diff --git a/x-pack/plugin/repositories-stats/qa/s3/build.gradle b/x-pack/plugin/repositories-stats/qa/s3/build.gradle index a75d7550d09cb..1c0ed18b01db1 100644 --- a/x-pack/plugin/repositories-stats/qa/s3/build.gradle +++ b/x-pack/plugin/repositories-stats/qa/s3/build.gradle @@ -44,10 +44,8 @@ if (useFixture) { integTest { dependsOn repositoryPlugin.bundlePlugin - runner { - systemProperty 'test.s3.bucket', s3Bucket - nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_repositories_metering" + BuildParams.testSeed : 'base_path' - } + systemProperty 'test.s3.bucket', s3Bucket + nonInputProperties.systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_repositories_metering" + BuildParams.testSeed : 'base_path' } testClusters.integTest { From 767edd68fac64acef60fef76138f57ef12a79409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 4 Aug 2020 21:37:00 +0200 Subject: [PATCH 04/19] Address first round of review comments --- .../repositories/azure/AzureRepository.java | 21 ++--- .../gcs/GoogleCloudStorageRepository.java | 14 +--- .../repositories/s3/S3Repository.java | 20 ++--- .../repositories/RepositoriesService.java | 28 ++++--- .../RepositoriesStatsArchive.java | 56 ++++++++----- .../repositories/Repository.java | 5 +- .../repositories/RepositoryInfo.java | 48 ++++++----- .../repositories/RepositoryStats.java | 7 ++ .../repositories/RepositoryStatsSnapshot.java | 32 ++++--- .../blobstore/BlobStoreRepository.java | 3 +- .../blobstore/MeteredBlobStoreRepository.java | 34 +++++--- .../RepositoriesServiceTests.java | 84 +++++++++++++------ .../RepositoriesStatsArchiveTests.java | 83 +++++++++++------- ...ESMockAPIBasedRepositoryIntegTestCase.java | 7 +- ...tractRepositoriesStatsAPIRestTestCase.java | 12 +-- .../RepositoriesStatsResponseTests.java | 9 +- .../TransportRepositoryStatsAction.java | 6 +- 17 files changed, 269 insertions(+), 200 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 90393db05b2fe..7b51c08629f35 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -75,7 +75,6 @@ public static final class Repository { private final ByteSizeValue chunkSize; private final AzureStorageService storageService; private final boolean readonly; - private final String container; public AzureRepository( final RepositoryMetadata metadata, @@ -83,7 +82,12 @@ public AzureRepository( final AzureStorageService storageService, final ClusterService clusterService, final RecoverySettings recoverySettings) { - super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildBasePath(metadata)); + super(metadata, + namedXContentRegistry, + clusterService, + recoverySettings, + buildBasePath(metadata), + Repository.CONTAINER_SETTING.get(metadata.settings())); this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings()); this.storageService = storageService; @@ -95,7 +99,6 @@ public AzureRepository( } else { this.readonly = locationMode == LocationMode.SECONDARY_ONLY; } - this.container = Repository.CONTAINER_SETTING.get(metadata.settings()); } private static BlobPath buildBasePath(RepositoryMetadata metadata) { @@ -136,16 +139,4 @@ protected ByteSizeValue chunkSize() { public boolean isReadOnly() { return readonly; } - - @Override - protected String location() { - BlobPath location = BlobPath.cleanPath(); - - location = location.add(container); - for (String path : basePath()) { - location = location.add(path); - } - - return location.buildAsString(); - } } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index a65c0a0d54242..5565a1d86a765 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -72,7 +72,7 @@ class GoogleCloudStorageRepository extends MeteredBlobStoreRepository { final GoogleCloudStorageService storageService, final ClusterService clusterService, final RecoverySettings recoverySettings) { - super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildBasePath(metadata)); + super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildBasePath(metadata), getSetting(BUCKET, metadata)); this.storageService = storageService; this.chunkSize = getSetting(CHUNK_SIZE, metadata); @@ -118,16 +118,4 @@ static T getSetting(Setting setting, RepositoryMetadata metadata) { } return value; } - - @Override - protected String location() { - BlobPath location = BlobPath.cleanPath(); - - location = location.add(bucket); - for (String path : basePath()) { - location = location.add(path); - } - - return location.buildAsString(); - } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index bb08bfad976eb..e80236e4501e8 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -194,7 +194,12 @@ class S3Repository extends MeteredBlobStoreRepository { final S3Service service, final ClusterService clusterService, final RecoverySettings recoverySettings) { - super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildBasePath(metadata)); + super(metadata, + namedXContentRegistry, + clusterService, + recoverySettings, + buildBasePath(metadata), + BUCKET_SETTING.get(metadata.settings())); this.service = service; // Parse and validate the user's S3 Storage Class setting @@ -328,17 +333,4 @@ protected void doClose() { } super.doClose(); } - - @Override - protected String location() { - logger.info("LOCATION {}", bucket); - BlobPath location = BlobPath.cleanPath(); - - location = location.add(bucket); - for (String path : basePath()) { - location = location.add(path); - } - - return location.buildAsString(); - } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 7dff66814d4b8..74d4434c499f1 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -48,6 +48,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -58,7 +59,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -72,6 +72,9 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C public static final Setting REPOSITORIES_STATS_ARCHIVE_RETENTION_PERIOD = Setting.positiveTimeSetting("repositories.stats.archive.retention_period", TimeValue.timeValueHours(2), Setting.Property.NodeScope); + public static final Setting REPOSITORIES_STATS_ARCHIVE_MAX_ARCHIVED_STATS = + Setting.intSetting("repositories.stats.archive.max_archived_stats", 100, 0, Setting.Property.NodeScope); + private final Map typesRegistry; private final Map internalTypesRegistry; @@ -98,7 +101,10 @@ public RepositoriesService(Settings settings, ClusterService clusterService, Tra clusterService.addHighPriorityApplier(this); } this.verifyAction = new VerifyNodeRepositoryAction(transportService, clusterService, this); - this.repositoriesStatsArchive = new RepositoriesStatsArchive(REPOSITORIES_STATS_ARCHIVE_RETENTION_PERIOD.get(settings)); + this.repositoriesStatsArchive = new RepositoriesStatsArchive(REPOSITORIES_STATS_ARCHIVE_RETENTION_PERIOD.get(settings), + REPOSITORIES_STATS_ARCHIVE_MAX_ARCHIVED_STATS.get(settings), + threadPool::relativeTimeInMillis, + threadPool::absoluteTimeInMillis); } /** @@ -418,14 +424,14 @@ public List repositoriesStats() { private List getRepositoryStatsForActiveRepositories() { return Stream.concat(repositories.values().stream(), internalRepositories.values().stream()) - .map(Repository::statsSnapshot) - .filter(Optional::isPresent) - .map(Optional::get) + .filter(r -> r instanceof MeteredBlobStoreRepository) + .map(r -> (MeteredBlobStoreRepository) r) + .map(MeteredBlobStoreRepository::statsSnapshot) .collect(Collectors.toList()); } - public void clearRepositoriesStatsArchive() { - repositoriesStatsArchive.clear(); + public List clearRepositoriesStatsArchive() { + return repositoriesStatsArchive.clear(); } public void registerInternalRepository(String name, String type) { @@ -460,9 +466,11 @@ private void closeRepository(Repository repository) { private void closeRepository(Repository repository, boolean archiveStats) { logger.debug("closing repository [{}][{}]", repository.getMetadata().type(), repository.getMetadata().name()); repository.close(); - if (archiveStats) { - Optional stats = repository.statsSnapshot(); - stats.ifPresent(repositoriesStatsArchive::archive); + if (archiveStats && repository instanceof MeteredBlobStoreRepository) { + RepositoryStatsSnapshot stats = ((MeteredBlobStoreRepository) repository).statsSnapshot(); + if (repositoriesStatsArchive.archive(stats) == false) { + logger.warn("Unable to archive the repository stats [{}] as the archive is full.", stats); + } } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java index 510f4d630382d..b60d40dc74380 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java @@ -21,26 +21,46 @@ import org.elasticsearch.common.unit.TimeValue; -import java.time.Duration; -import java.time.Instant; import java.util.ArrayDeque; import java.util.Deque; import java.util.List; +import java.util.function.LongSupplier; public final class RepositoriesStatsArchive { private final TimeValue retentionPeriod; + private final int maxCapacity; + private final LongSupplier relativeTimeSupplier; + private final LongSupplier absoluteTimeSupplier; private final Deque archive = new ArrayDeque<>(); - RepositoriesStatsArchive(TimeValue retentionPeriod) { + public RepositoriesStatsArchive(TimeValue retentionPeriod, + int maxCapacity, + LongSupplier relativeTimeSupplier, + LongSupplier absoluteTimeSupplier) { this.retentionPeriod = retentionPeriod; + this.maxCapacity = maxCapacity; + this.relativeTimeSupplier = relativeTimeSupplier; + this.absoluteTimeSupplier = absoluteTimeSupplier; } - synchronized void archive(RepositoryStatsSnapshot repositoryStats) { - if (repositoryStats.getRepositoryInfo().isStopped() == false) { - repositoryStats = repositoryStats.withStoppedRepo(); - } - archive.add(repositoryStats); + /** + * Archives the specified repository stats snapshot into the archive + * if it's possible without violating the capacity constraints. + * + * @return {@code true} if the repository stats were archived, {@code false} otherwise. + */ + synchronized boolean archive(RepositoryStatsSnapshot repositoryStats) { evict(); + + if (archive.size() >= maxCapacity) { + return false; + } + + RepositoryInfo stoppedRepoInfo = + repositoryStats.getRepositoryInfo().stopped(absoluteTimeSupplier.getAsLong()); + repositoryStats = + new RepositoryStatsSnapshot(stoppedRepoInfo, repositoryStats.getRepositoryStats(), relativeTimeSupplier.getAsLong()); + return archive.add(repositoryStats); } synchronized List getArchivedStats() { @@ -48,23 +68,21 @@ synchronized List getArchivedStats() { return List.copyOf(archive); } - synchronized void clear() { + /** + * Clears the archive, returning the valid archived entries up until that point. + * + * @return the repository stats that were stored before clearing the archive. + */ + synchronized List clear() { + List archivedStats = getArchivedStats(); archive.clear(); + return archivedStats; } private void evict() { - Instant retentionDeadline = getRetentionDeadline(); RepositoryStatsSnapshot stats; - while ((stats = archive.peek()) != null && shouldEvict(stats, retentionDeadline)) { + while ((stats = archive.peek()) != null && stats.ageInMillis(relativeTimeSupplier) >= retentionPeriod.getMillis()) { archive.poll(); } } - - private boolean shouldEvict(RepositoryStatsSnapshot stats, Instant deadline) { - return stats.wasRepoStoppedBefore(deadline); - } - - private Instant getRetentionDeadline() { - return Instant.now().minus(Duration.ofMillis(retentionPeriod.getMillis())); - } } diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 36ad304402720..ec0fb5b561ad1 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -41,7 +41,6 @@ import java.io.IOException; import java.util.Collection; import java.util.Map; -import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; @@ -155,8 +154,8 @@ void deleteSnapshots(Collection snapshotIds, long repositoryStateId, /** * Returns stats on the repository usage */ - default Optional statsSnapshot() { - return Optional.empty(); + default RepositoryStats stats() { + return RepositoryStats.EMPTY_STATS; } /** diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java index 260619996ac9d..c41f5bfe74d2f 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; -import java.time.Instant; import java.util.Objects; public final class RepositoryInfo implements Writeable, ToXContentFragment { @@ -35,15 +34,15 @@ public final class RepositoryInfo implements Writeable, ToXContentFragment { public final String name; public final String type; public final String location; - public final Instant startedAt; + public final long startedAt; @Nullable - public final Instant stoppedAt; + public final Long stoppedAt; public RepositoryInfo(String ephemeralId, String name, String type, String location, - Instant startedAt) { + Long startedAt) { this(ephemeralId, name, type, location, startedAt, null); } @@ -51,14 +50,14 @@ public RepositoryInfo(String ephemeralId, String name, String type, String location, - Instant startedAt, - @Nullable Instant stoppedAt) { + Long startedAt, + @Nullable Long stoppedAt) { this.ephemeralId = ephemeralId; this.name = name; this.type = type; this.location = location; this.startedAt = startedAt; - if (stoppedAt != null && startedAt.isAfter(stoppedAt)) { + if (stoppedAt != null && startedAt > stoppedAt) { throw new IllegalArgumentException("createdAt must be before or equal to stoppedAt"); } this.stoppedAt = stoppedAt; @@ -69,21 +68,14 @@ public RepositoryInfo(StreamInput in) throws IOException { this.name = in.readString(); this.type = in.readString(); this.location = in.readString(); - this.startedAt = in.readInstant(); - this.stoppedAt = in.readOptionalInstant(); + this.startedAt = in.readLong(); + this.stoppedAt = in.readOptionalLong(); } - public RepositoryInfo stopped() { + public RepositoryInfo stopped(long stoppedAt) { assert isStopped() == false : "The repository is already stopped"; - return new RepositoryInfo(ephemeralId, name, type, location, startedAt, Instant.now()); - } - - public boolean wasStoppedBefore(Instant instant) { - if (stoppedAt == null) { - return false; - } - return stoppedAt.isBefore(instant); + return new RepositoryInfo(ephemeralId, name, type, location, startedAt, stoppedAt); } public boolean isStopped() { @@ -96,8 +88,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeString(type); out.writeString(location); - out.writeInstant(startedAt); - out.writeOptionalInstant(stoppedAt); + out.writeLong(startedAt); + out.writeOptionalLong(stoppedAt); } @Override @@ -106,9 +98,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("repository_type", type); builder.field("repository_location", location); builder.field("repository_ephemeral_id", ephemeralId); - builder.field("repository_started_at", startedAt.toEpochMilli()); + builder.field("repository_started_at", startedAt); if (stoppedAt != null) { - builder.field("repository_stopped_at", stoppedAt.toEpochMilli()); + builder.field("repository_stopped_at", stoppedAt); } return builder; } @@ -130,4 +122,16 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(ephemeralId, name, type, location, startedAt, stoppedAt); } + + @Override + public String toString() { + return "RepositoryInfo{" + + "ephemeralId='" + ephemeralId + '\'' + + ", name='" + name + '\'' + + ", type='" + type + '\'' + + ", location='" + location + '\'' + + ", startedAt=" + startedAt + + ", stoppedAt=" + stoppedAt + + '}'; + } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java index 53ed90f828b2c..e4fe80510a549 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java @@ -69,4 +69,11 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(requestCounts); } + + @Override + public String toString() { + return "RepositoryStats{" + + "requestCounts=" + requestCounts + + '}'; + } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java index e5b9062340ee6..809ac3ddbef7a 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java @@ -19,6 +19,7 @@ package org.elasticsearch.repositories; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -26,22 +27,27 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; -import java.time.Instant; import java.util.Objects; +import java.util.function.LongSupplier; public final class RepositoryStatsSnapshot implements Writeable, ToXContent { private final RepositoryInfo repositoryInfo; private final RepositoryStats repositoryStats; + @Nullable + private final Long createdAtMillis; public RepositoryStatsSnapshot(RepositoryInfo repositoryInfo, - RepositoryStats repositoryStats) { + RepositoryStats repositoryStats, + @Nullable Long createdAtMillis) { this.repositoryInfo = repositoryInfo; this.repositoryStats = repositoryStats; + this.createdAtMillis = createdAtMillis; } public RepositoryStatsSnapshot(StreamInput in) throws IOException { this.repositoryInfo = new RepositoryInfo(in); this.repositoryStats = new RepositoryStats(in); + this.createdAtMillis = null; } public RepositoryInfo getRepositoryInfo() { @@ -52,12 +58,8 @@ public RepositoryStats getRepositoryStats() { return repositoryStats; } - public boolean wasRepoStoppedBefore(Instant instant) { - return repositoryInfo.wasStoppedBefore(instant); - } - - public RepositoryStatsSnapshot withStoppedRepo() { - return new RepositoryStatsSnapshot(repositoryInfo.stopped(), repositoryStats); + public long ageInMillis(LongSupplier relativeTimeInMillis) { + return createdAtMillis == null ? 0 : relativeTimeInMillis.getAsLong() - createdAtMillis; } @Override @@ -81,11 +83,21 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; RepositoryStatsSnapshot that = (RepositoryStatsSnapshot) o; return Objects.equals(repositoryInfo, that.repositoryInfo) && - Objects.equals(repositoryStats, that.repositoryStats); + Objects.equals(repositoryStats, that.repositoryStats) && + Objects.equals(createdAtMillis, that.createdAtMillis); } @Override public int hashCode() { - return Objects.hash(repositoryInfo, repositoryStats); + return Objects.hash(repositoryInfo, repositoryStats, createdAtMillis); + } + + @Override + public String toString() { + return "RepositoryStatsSnapshot{" + + "repositoryInfo=" + repositoryInfo + + ", repositoryStats=" + repositoryStats + + ", createdAtMillis=" + createdAtMillis + + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index c6c1d303a1d98..dab035e5bb177 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -542,7 +542,8 @@ public RepositoryMetadata getMetadata() { return metadata; } - protected RepositoryStats stats() { + @Override + public RepositoryStats stats() { final BlobStore store = blobStore.get(); if (store == null) { return RepositoryStats.EMPTY_STATS; diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java index f01205afb7dec..b91c76dac4dd0 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java @@ -27,29 +27,37 @@ import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoryInfo; import org.elasticsearch.repositories.RepositoryStatsSnapshot; - -import java.time.Instant; -import java.util.Optional; +import org.elasticsearch.threadpool.ThreadPool; public abstract class MeteredBlobStoreRepository extends BlobStoreRepository { - private final String ephemeralId; - private final Instant startedAt; + private final RepositoryInfo repositoryInfo; public MeteredBlobStoreRepository(RepositoryMetadata metadata, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, RecoverySettings recoverySettings, - BlobPath basePath) { + BlobPath basePath, + String bucket) { super(metadata, namedXContentRegistry, clusterService, recoverySettings, basePath); - this.ephemeralId = UUIDs.randomBase64UUID(); - this.startedAt = Instant.now(); + ThreadPool threadPool = clusterService.getClusterApplierService().threadPool(); + this.repositoryInfo = new RepositoryInfo(UUIDs.randomBase64UUID(), + metadata.name(), + metadata.type(), + getLocation(basePath, bucket), + threadPool.absoluteTimeInMillis()); } - @Override - public Optional statsSnapshot() { - RepositoryInfo repositoryInfo = new RepositoryInfo(ephemeralId, metadata.name(), metadata.type(), location(), startedAt); - return Optional.of(new RepositoryStatsSnapshot(repositoryInfo, stats())); + public RepositoryStatsSnapshot statsSnapshot() { + return new RepositoryStatsSnapshot(repositoryInfo, stats(), threadPool.relativeTimeInMillis()); } - protected abstract String location(); + private static String getLocation(BlobPath basePath, String bucket) { + BlobPath location = BlobPath.cleanPath(); + + location = location.add(bucket); + for (String path : basePath) { + location = location.add(path); + } + return location.buildAsString(); + } } diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index a65da23c4c46b..be2456e3b0d3f 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -29,17 +29,23 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterApplierService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; +import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.test.ESTestCase; @@ -47,17 +53,16 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; -import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class RepositoriesServiceTests extends ESTestCase { @@ -71,8 +76,14 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null, Collections.emptySet()); - Map typesRegistry = Map.of(TestRepository.TYPE, TestRepository::new, - SecondTestRepository.TYPE, SecondTestRepository::new); + final ClusterApplierService clusterApplierService = mock(ClusterApplierService.class); + when(clusterApplierService.threadPool()).thenReturn(threadPool); + final ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterApplierService()).thenReturn(clusterApplierService); + Map typesRegistry = + Map.of(TestRepository.TYPE, TestRepository::new, + MeteredRepositoryTypeA.TYPE, metadata -> new MeteredRepositoryTypeA(metadata, clusterService), + MeteredRepositoryTypeB.TYPE, metadata -> new MeteredRepositoryTypeB(metadata, clusterService)); repositoriesService = new RepositoriesService(Settings.EMPTY, mock(ClusterService.class), transportService, Collections.emptyMap(), typesRegistry, threadPool); repositoriesService.start(); @@ -123,23 +134,23 @@ public void testRegisterRejectsInvalidRepositoryNames() { public void testRepositoriesStatsCanHaveTheSameNameAndDifferentTypeOverTime() { String repoName = "name"; expectThrows(RepositoryMissingException.class, () -> repositoriesService.repository(repoName)); - repositoriesService.registerInternalRepository(repoName, TestRepository.TYPE); + repositoriesService.registerInternalRepository(repoName, MeteredRepositoryTypeA.TYPE); assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); repositoriesService.unregisterInternalRepository(repoName); assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); - repositoriesService.registerInternalRepository(repoName, SecondTestRepository.TYPE); + repositoriesService.registerInternalRepository(repoName, MeteredRepositoryTypeB.TYPE); List repositoriesStats = repositoriesService.repositoriesStats(); assertThat(repositoriesStats.size(), equalTo(2)); - RepositoryStatsSnapshot repositoryStats = repositoriesStats.get(0); - assertThat(repositoryStats.getRepositoryInfo().type, equalTo(TestRepository.TYPE)); - assertThat(repositoryStats.getRepositoryStats().requestCounts, equalTo(Map.of("TEST-REPO", 10L))); + RepositoryStatsSnapshot repositoryStatsTypeA = repositoriesStats.get(0); + assertThat(repositoryStatsTypeA.getRepositoryInfo().type, equalTo(MeteredRepositoryTypeA.TYPE)); + assertThat(repositoryStatsTypeA.getRepositoryStats(), equalTo(MeteredRepositoryTypeA.STATS)); - RepositoryStatsSnapshot repositoryStats2 = repositoriesStats.get(1); - assertThat(repositoryStats2.getRepositoryInfo().type, equalTo(SecondTestRepository.TYPE)); - assertThat(repositoryStats2.getRepositoryStats().requestCounts, equalTo(Map.of("SECOND-TEST", 20L))); + RepositoryStatsSnapshot repositoryStatsTypeB = repositoriesStats.get(1); + assertThat(repositoryStatsTypeB.getRepositoryInfo().type, equalTo(MeteredRepositoryTypeB.TYPE)); + assertThat(repositoryStatsTypeB.getRepositoryStats(), equalTo(MeteredRepositoryTypeB.STATS)); } private void assertThrowsOnRegister(String repoName) { @@ -150,7 +161,6 @@ private void assertThrowsOnRegister(String repoName) { private static class TestRepository implements Repository { private static final String TYPE = "internal"; - private static final String LOCATION = "location"; private boolean isClosed; private boolean isStarted; @@ -285,29 +295,53 @@ public void stop() { public void close() { isClosed = true; } + } + + private static class MeteredRepositoryTypeA extends MeteredBlobStoreRepository { + private static final String TYPE = "type-a"; + private static final RepositoryStats STATS = new RepositoryStats(Map.of("GET", 10L)); + + private MeteredRepositoryTypeA(RepositoryMetadata metadata, ClusterService clusterService) { + super(metadata, + mock(NamedXContentRegistry.class), + clusterService, + mock(RecoverySettings.class), + BlobPath.cleanPath(), + "bucket-a"); + } @Override - public Optional statsSnapshot() { - RepositoryInfo repositoryInfo = - new RepositoryInfo(UUIDs.randomBase64UUID(), metadata.name(), metadata.type(), LOCATION, Instant.now()); - return Optional.of(new RepositoryStatsSnapshot(repositoryInfo, getRepositoryStats())); + protected BlobStore createBlobStore() { + return mock(BlobStore.class); } - protected RepositoryStats getRepositoryStats() { - return new RepositoryStats(Map.of("TEST-REPO", 10L)); + @Override + public RepositoryStats stats() { + return STATS; } } - private static class SecondTestRepository extends TestRepository { - private static final String TYPE = "second-internal"; + private static class MeteredRepositoryTypeB extends MeteredBlobStoreRepository { + private static final String TYPE = "type-b"; + private static final RepositoryStats STATS = new RepositoryStats(Map.of("LIST", 20L)); + + private MeteredRepositoryTypeB(RepositoryMetadata metadata, ClusterService clusterService) { + super(metadata, + mock(NamedXContentRegistry.class), + clusterService, + mock(RecoverySettings.class), + BlobPath.cleanPath(), + "bucket-b"); + } - private SecondTestRepository(RepositoryMetadata metadata) { - super(metadata); + @Override + protected BlobStore createBlobStore() { + return mock(BlobStore.class); } @Override - protected RepositoryStats getRepositoryStats() { - return new RepositoryStats(Map.of("SECOND-TEST", 20L)); + public RepositoryStats stats() { + return STATS; } } } diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java index 5fcf055a9936e..46ed159352c0c 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java @@ -23,37 +23,32 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; -import java.time.Duration; -import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import static org.hamcrest.Matchers.equalTo; public class RepositoriesStatsArchiveTests extends ESTestCase { public void testStatsAreEvictedOnceTheyAreOlderThanRetentionPeriod() { - int retentionTimeInHours = randomIntBetween(1, 4); - RepositoriesStatsArchive repositoriesStatsArchive = - new RepositoriesStatsArchive(TimeValue.timeValueHours(retentionTimeInHours)); - - int statsOlderThanRetentionPeriodCount = randomInt(10); - for (int i = 0; i < statsOlderThanRetentionPeriodCount; i++) { - RepositoryInfo repositoryInfo = - createRepositoryInfo(retentionTimeInHours + 2, retentionTimeInHours + 1); + int retentionTimeInMillis = randomIntBetween(100, 1000); - RepositoryStatsSnapshot repoStats = - new RepositoryStatsSnapshot(repositoryInfo, RepositoryStats.EMPTY_STATS); + AtomicLong fakeRelativeClock = new AtomicLong(); + RepositoriesStatsArchive repositoriesStatsArchive = + new RepositoriesStatsArchive(TimeValue.timeValueMillis(retentionTimeInMillis), + 100, + fakeRelativeClock::get, + System::currentTimeMillis); + for (int i = 0; i < randomInt(10); i++) { + RepositoryStatsSnapshot repoStats = createRepositoryStats(RepositoryStats.EMPTY_STATS); repositoriesStatsArchive.archive(repoStats); } + fakeRelativeClock.set(retentionTimeInMillis * 2); int statsToBeRetainedCount = randomInt(10); for (int i = 0; i < statsToBeRetainedCount; i++) { - RepositoryInfo repositoryInfo = - createRepositoryInfo(0); - - RepositoryStatsSnapshot repoStats = - new RepositoryStatsSnapshot(repositoryInfo, new RepositoryStats(Map.of("GET", 10L))); + RepositoryStatsSnapshot repoStats = createRepositoryStats(new RepositoryStats(Map.of("GET", 10L))); repositoriesStatsArchive.archive(repoStats); } @@ -64,22 +59,52 @@ public void testStatsAreEvictedOnceTheyAreOlderThanRetentionPeriod() { } } - private RepositoryInfo createRepositoryInfo(int hoursSinceRepoStarted) { - return new RepositoryInfo(UUIDs.randomBase64UUID(), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - Instant.now().minus(Duration.ofHours(hoursSinceRepoStarted)), - null); + public void testStatsAreRejectedIfTheArchiveIsFull() { + int retentionTimeInMillis = randomIntBetween(100, 1000); + + AtomicLong fakeRelativeClock = new AtomicLong(); + RepositoriesStatsArchive repositoriesStatsArchive = + new RepositoriesStatsArchive(TimeValue.timeValueMillis(retentionTimeInMillis), + 1, + fakeRelativeClock::get, + System::currentTimeMillis); + + assertTrue(repositoriesStatsArchive.archive(createRepositoryStats(RepositoryStats.EMPTY_STATS))); + + fakeRelativeClock.set(retentionTimeInMillis * 2); + // Now there's room since the previous stats should be evicted + assertTrue(repositoriesStatsArchive.archive(createRepositoryStats(RepositoryStats.EMPTY_STATS))); + // There's no room for stats with the same creation time + assertFalse(repositoriesStatsArchive.archive(createRepositoryStats(RepositoryStats.EMPTY_STATS))); + } + + public void testClearArchive() { + int retentionTimeInMillis = randomIntBetween(100, 1000); + AtomicLong fakeRelativeClock = new AtomicLong(); + RepositoriesStatsArchive repositoriesStatsArchive = + new RepositoriesStatsArchive(TimeValue.timeValueMillis(retentionTimeInMillis), + 100, + fakeRelativeClock::get, + System::currentTimeMillis); + + int archivedStats = randomInt(20); + for (int i = 0; i < archivedStats; i++) { + repositoriesStatsArchive.archive(createRepositoryStats(RepositoryStats.EMPTY_STATS)); + } + + List removedStats = repositoriesStatsArchive.clear(); + assertThat(removedStats.size(), equalTo(archivedStats)); + + assertThat(repositoriesStatsArchive.getArchivedStats().size(), equalTo(0)); } - private RepositoryInfo createRepositoryInfo(int hoursSinceRepoStarted, int hoursSinceRepoStopped) { - assert hoursSinceRepoStarted > hoursSinceRepoStopped; - return new RepositoryInfo(UUIDs.randomBase64UUID(), + private RepositoryStatsSnapshot createRepositoryStats(RepositoryStats repositoryStats) { + RepositoryInfo repositoryInfo = new RepositoryInfo(UUIDs.randomBase64UUID(), randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), - Instant.now().minus(Duration.ofHours(hoursSinceRepoStarted)), - Instant.now().minus(Duration.ofHours(hoursSinceRepoStopped))); + System.currentTimeMillis(), + null); + return new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, null); } } diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java index 69ee1fdd3c8ef..03a69db208530 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java @@ -39,7 +39,6 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryStats; -import org.elasticsearch.repositories.RepositoryStatsSnapshot; import org.elasticsearch.test.BackgroundIndexer; import org.junit.After; import org.junit.AfterClass; @@ -54,7 +53,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -213,10 +211,7 @@ public void testRequestStats() throws Exception { } }) .filter(Objects::nonNull) - .map(Repository::statsSnapshot) - .filter(Optional::isPresent) - .map(Optional::get) - .map(RepositoryStatsSnapshot::getRepositoryStats) + .map(Repository::stats) .reduce(RepositoryStats::merge) .get(); diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java index eac8bd47780f7..184f7d01f890f 100644 --- a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java +++ b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java @@ -23,7 +23,6 @@ import org.junit.Before; import java.io.IOException; -import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -315,7 +314,7 @@ private List getRepositoriesStats() throws IOException .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().longValue())); RepositoryStats repositoryStats = new RepositoryStats(requestCounters); - RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot(repositoryInfo, repositoryStats); + RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, null); repositoriesStats.add(statsSnapshot); } } @@ -329,14 +328,7 @@ private RepositoryInfo parseRepositoryInfo(Map nodeStatSnapshot) String location = extractValue(nodeStatSnapshot, "repository_location"); Long startedAt = extractValue(nodeStatSnapshot, "repository_started_at"); Long stoppedAt = extractValue(nodeStatSnapshot, "repository_stopped_at"); - return new RepositoryInfo( - id, - name, - type, - location, - Instant.ofEpochMilli(startedAt), - stoppedAt != null ? Instant.ofEpochMilli(stoppedAt) : null - ); + return new RepositoryInfo(id, name, type, location, startedAt, stoppedAt); } private Set getNodeIds() throws IOException { diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java index 41020f1900c05..689e654322fd3 100644 --- a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java +++ b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java @@ -15,8 +15,6 @@ import org.elasticsearch.repositories.RepositoryStatsSnapshot; import org.elasticsearch.test.ESTestCase; -import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -71,12 +69,13 @@ private RepositoriesStatsResponse createResponse() { String repoName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); String repoType = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); String repoLocation = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); - Instant startedAt = Instant.now().minus(Duration.ofMinutes(randomLongBetween(1, 30))); - Instant stoppedAt = randomBoolean() ? Instant.now() : null; + Long startedAt = System.currentTimeMillis() - 1; + Long stoppedAt = randomBoolean() ? System.currentTimeMillis() : null; RepositoryInfo repositoryInfo = new RepositoryInfo(repoId, repoName, repoType, repoLocation, startedAt, stoppedAt); RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot( repositoryInfo, - new RepositoryStats(Map.of("GET", randomLongBetween(0, 2000))) + new RepositoryStats(Map.of("GET", randomLongBetween(0, 2000))), + null ); nodeRepoStats.add(statsSnapshot); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportRepositoryStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportRepositoryStatsAction.java index 660b780c2a114..2c1300820a4a2 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportRepositoryStatsAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportRepositoryStatsAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryStats; -import org.elasticsearch.repositories.RepositoryStatsSnapshot; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -85,9 +84,6 @@ protected RepositoryStatsNodeResponse nodeOperation(RepositoryStatsNodeRequest r return new RepositoryStatsNodeResponse(clusterService.localNode(), RepositoryStats.EMPTY_STATS); } final Repository repository = repositoriesService.repository(request.getRepository()); - RepositoryStats repositoryStats = repository.statsSnapshot() - .map(RepositoryStatsSnapshot::getRepositoryStats) - .orElse(RepositoryStats.EMPTY_STATS); - return new RepositoryStatsNodeResponse(clusterService.localNode(), repositoryStats); + return new RepositoryStatsNodeResponse(clusterService.localNode(), repository.stats()); } } From 02c5d93e02cb856f4b2cca5df4d2ea35a953e968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Wed, 5 Aug 2020 10:58:15 +0200 Subject: [PATCH 05/19] Simplify TransportActions and return repositories stats during archive clearing --- .../RepositoriesStatsArchive.java | 20 +++++-- .../ClearRepositoriesStatsArchiveAction.java | 4 +- ...arRepositoriesStatsArchiveNodeRequest.java | 20 ------- ...rRepositoriesStatsArchiveNodeResponse.java | 23 -------- ...ClearRepositoriesStatsArchiveResponse.java | 59 ------------------- .../action/RepositoriesNodeStatsRequest.java | 20 ------- ...rtClearRepositoriesStatsArchiveAction.java | 34 +++++++---- .../TransportRepositoriesStatsAction.java | 11 +++- ...stClearRepositoriesStatsArchiveAction.java | 4 +- ...tractRepositoriesStatsAPIRestTestCase.java | 22 +++---- 10 files changed, 64 insertions(+), 153 deletions(-) delete mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeRequest.java delete mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeResponse.java delete mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveResponse.java delete mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsRequest.java diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java index b60d40dc74380..fa86d306d5317 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java @@ -19,6 +19,8 @@ package org.elasticsearch.repositories; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.common.unit.TimeValue; import java.util.ArrayDeque; @@ -27,6 +29,8 @@ import java.util.function.LongSupplier; public final class RepositoriesStatsArchive { + private static final Logger logger = LogManager.getLogger(RepositoriesStatsArchive.class); + private final TimeValue retentionPeriod; private final int maxCapacity; private final LongSupplier relativeTimeSupplier; @@ -49,7 +53,9 @@ public RepositoriesStatsArchive(TimeValue retentionPeriod, * * @return {@code true} if the repository stats were archived, {@code false} otherwise. */ - synchronized boolean archive(RepositoryStatsSnapshot repositoryStats) { + synchronized boolean archive(final RepositoryStatsSnapshot repositoryStats) { + assert containsRepositoryStats(repositoryStats) == false + : "A repository with ephemeral id " + repositoryStats.getRepositoryInfo().ephemeralId + " is already archived"; evict(); if (archive.size() >= maxCapacity) { @@ -58,9 +64,9 @@ synchronized boolean archive(RepositoryStatsSnapshot repositoryStats) { RepositoryInfo stoppedRepoInfo = repositoryStats.getRepositoryInfo().stopped(absoluteTimeSupplier.getAsLong()); - repositoryStats = + RepositoryStatsSnapshot stoppedStats = new RepositoryStatsSnapshot(stoppedRepoInfo, repositoryStats.getRepositoryStats(), relativeTimeSupplier.getAsLong()); - return archive.add(repositoryStats); + return archive.add(stoppedStats); } synchronized List getArchivedStats() { @@ -76,13 +82,19 @@ synchronized List getArchivedStats() { synchronized List clear() { List archivedStats = getArchivedStats(); archive.clear(); + logger.debug("RepositoriesStatsArchive have been cleared. Removed stats: [{}]", archivedStats); return archivedStats; } private void evict() { RepositoryStatsSnapshot stats; while ((stats = archive.peek()) != null && stats.ageInMillis(relativeTimeSupplier) >= retentionPeriod.getMillis()) { - archive.poll(); + RepositoryStatsSnapshot removedStats = archive.poll(); + logger.debug("Evicting repository stats [{}]", removedStats); } } + + private boolean containsRepositoryStats(RepositoryStatsSnapshot repositoryStats) { + return archive.stream().anyMatch(r -> r.getRepositoryInfo().ephemeralId.equals(repositoryStats.getRepositoryInfo().ephemeralId)); + } } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java index 317e37c034275..5ebf2f56bf12f 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java @@ -8,12 +8,12 @@ import org.elasticsearch.action.ActionType; -public final class ClearRepositoriesStatsArchiveAction extends ActionType { +public final class ClearRepositoriesStatsArchiveAction extends ActionType { public static final ClearRepositoriesStatsArchiveAction INSTANCE = new ClearRepositoriesStatsArchiveAction(); static final String NAME = "cluster:monitor/xpack/repositories_stats/clear_repositories_stats_archive"; ClearRepositoriesStatsArchiveAction() { - super(NAME, ClearRepositoriesStatsArchiveResponse::new); + super(NAME, RepositoriesStatsResponse::new); } } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeRequest.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeRequest.java deleted file mode 100644 index 5f9e8847cf33e..0000000000000 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeRequest.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.repositories.stats.action; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.transport.TransportRequest; - -import java.io.IOException; - -final class ClearRepositoriesStatsArchiveNodeRequest extends TransportRequest { - ClearRepositoriesStatsArchiveNodeRequest() {} - - ClearRepositoriesStatsArchiveNodeRequest(StreamInput in) throws IOException { - super(in); - } -} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeResponse.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeResponse.java deleted file mode 100644 index 47c93cabcb2d0..0000000000000 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveNodeResponse.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.repositories.stats.action; - -import org.elasticsearch.action.support.nodes.BaseNodeResponse; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.io.stream.StreamInput; - -import java.io.IOException; - -final class ClearRepositoriesStatsArchiveNodeResponse extends BaseNodeResponse { - ClearRepositoriesStatsArchiveNodeResponse(StreamInput in) throws IOException { - super(in); - } - - ClearRepositoriesStatsArchiveNodeResponse(DiscoveryNode node) { - super(node); - } -} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveResponse.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveResponse.java deleted file mode 100644 index b65c4ab5f1198..0000000000000 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveResponse.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.repositories.stats.action; - -import org.elasticsearch.action.FailedNodeException; -import org.elasticsearch.action.support.nodes.BaseNodesResponse; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.List; - -public class ClearRepositoriesStatsArchiveResponse extends BaseNodesResponse - implements - ToXContentObject { - - public ClearRepositoriesStatsArchiveResponse(StreamInput in) throws IOException { - super(in); - } - - public ClearRepositoriesStatsArchiveResponse( - ClusterName clusterName, - List nodes, - List failures - ) { - super(clusterName, nodes, failures); - } - - @Override - protected List readNodesFrom(StreamInput in) throws IOException { - return in.readList(ClearRepositoriesStatsArchiveNodeResponse::new); - } - - @Override - protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { - out.writeList(nodes); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (failures().isEmpty() == false) { - builder.startArray("failures"); - for (FailedNodeException failure : failures()) { - builder.startObject(); - failure.toXContent(builder, params); - builder.endObject(); - } - builder.endArray(); - } - return builder; - } -} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsRequest.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsRequest.java deleted file mode 100644 index e943efe807c33..0000000000000 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsRequest.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.repositories.stats.action; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.transport.TransportRequest; - -import java.io.IOException; - -final class RepositoriesNodeStatsRequest extends TransportRequest { - RepositoriesNodeStatsRequest() {} - - RepositoriesNodeStatsRequest(StreamInput in) throws IOException { - super(in); - } -} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java index 73ae0e6d31b7f..0edbbfffb430a 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java @@ -13,8 +13,10 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.RepositoryStatsSnapshot; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -22,9 +24,9 @@ public final class TransportClearRepositoriesStatsArchiveAction extends TransportNodesAction< ClearRepositoriesStatsArchiveRequest, - ClearRepositoriesStatsArchiveResponse, - ClearRepositoriesStatsArchiveNodeRequest, - ClearRepositoriesStatsArchiveNodeResponse> { + RepositoriesStatsResponse, + TransportClearRepositoriesStatsArchiveAction.ClearRepositoriesStatsArchiveNodeRequest, + RepositoriesNodeStatsResponse> { private final RepositoriesService repositoriesService; @@ -45,18 +47,18 @@ public TransportClearRepositoriesStatsArchiveAction( ClearRepositoriesStatsArchiveRequest::new, ClearRepositoriesStatsArchiveNodeRequest::new, ThreadPool.Names.SAME, - ClearRepositoriesStatsArchiveNodeResponse.class + RepositoriesNodeStatsResponse.class ); this.repositoriesService = repositoriesService; } @Override - protected ClearRepositoriesStatsArchiveResponse newResponse( + protected RepositoriesStatsResponse newResponse( ClearRepositoriesStatsArchiveRequest request, - List nodesResponses, + List nodesResponses, List failures ) { - return new ClearRepositoriesStatsArchiveResponse(clusterService.getClusterName(), nodesResponses, failures); + return new RepositoriesStatsResponse(clusterService.getClusterName(), nodesResponses, failures); } @Override @@ -65,13 +67,21 @@ protected ClearRepositoriesStatsArchiveNodeRequest newNodeRequest(ClearRepositor } @Override - protected ClearRepositoriesStatsArchiveNodeResponse newNodeResponse(StreamInput in) throws IOException { - return new ClearRepositoriesStatsArchiveNodeResponse(in); + protected RepositoriesNodeStatsResponse newNodeResponse(StreamInput in) throws IOException { + return new RepositoriesNodeStatsResponse(in); } @Override - protected ClearRepositoriesStatsArchiveNodeResponse nodeOperation(ClearRepositoriesStatsArchiveNodeRequest request, Task task) { - repositoriesService.clearRepositoriesStatsArchive(); - return new ClearRepositoriesStatsArchiveNodeResponse(clusterService.localNode()); + protected RepositoriesNodeStatsResponse nodeOperation(ClearRepositoriesStatsArchiveNodeRequest request, Task task) { + List clearedStats = repositoriesService.clearRepositoriesStatsArchive(); + return new RepositoriesNodeStatsResponse(clusterService.localNode(), clearedStats); + } + + static final class ClearRepositoriesStatsArchiveNodeRequest extends TransportRequest { + ClearRepositoriesStatsArchiveNodeRequest() {} + + ClearRepositoriesStatsArchiveNodeRequest(StreamInput in) throws IOException { + super(in); + } } } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java index ffa64b8723da4..35346cdc724f8 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -23,7 +24,7 @@ public final class TransportRepositoriesStatsAction extends TransportNodesAction< RepositoriesStatsRequest, RepositoriesStatsResponse, - RepositoriesNodeStatsRequest, + TransportRepositoriesStatsAction.RepositoriesNodeStatsRequest, RepositoriesNodeStatsResponse> { private final RepositoriesService repositoriesService; @@ -73,4 +74,12 @@ protected RepositoriesNodeStatsResponse newNodeResponse(StreamInput in) throws I protected RepositoriesNodeStatsResponse nodeOperation(RepositoriesNodeStatsRequest request, Task task) { return new RepositoriesNodeStatsResponse(clusterService.localNode(), repositoriesService.repositoriesStats()); } + + static final class RepositoriesNodeStatsRequest extends TransportRequest { + RepositoriesNodeStatsRequest() {} + + RepositoriesNodeStatsRequest(StreamInput in) throws IOException { + super(in); + } + } } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java index cf48e8fc41398..2bf3977c32a89 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java @@ -10,7 +10,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.xpack.repositories.stats.action.ClearRepositoriesStatsArchiveAction; import org.elasticsearch.xpack.repositories.stats.action.ClearRepositoriesStatsArchiveRequest; @@ -34,7 +34,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli return channel -> client.execute( ClearRepositoriesStatsArchiveAction.INSTANCE, clearArchivesRequest, - new RestToXContentListener<>(channel) + new RestActions.NodesResponseRestListener<>(channel) ); } } diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java index 184f7d01f890f..67631d2840217 100644 --- a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java +++ b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java @@ -19,7 +19,6 @@ import org.elasticsearch.repositories.RepositoryStatsSnapshot; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.rest.ESRestTestCase; -import org.junit.After; import org.junit.Before; import java.io.IOException; @@ -55,12 +54,7 @@ public abstract class AbstractRepositoriesStatsAPIRestTestCase extends ESRestTes protected abstract List writeCounterKeys(); @Before - public void clear() throws Exception { - clearRepositoriesStats(); - } - - @After - public void clearStats() throws Exception { + public void clearArchive() throws Exception { clearRepositoriesStats(); } @@ -110,9 +104,12 @@ public void testClearRepositoriesStats() throws Exception { snapshotAndRestoreIndex((repository, index) -> { deleteRepository(repository); - assertThat(getRepositoriesStats().size(), equalTo(1)); + List repositoriesStatsBeforeClearing = getRepositoriesStats(); + assertThat(repositoriesStatsBeforeClearing.size(), equalTo(1)); - clearRepositoriesStats(); + List removedRepositoriesStats = clearRepositoriesStats(); + + assertThat(repositoriesStatsBeforeClearing, equalTo(removedRepositoriesStats)); assertThat(getRepositoriesStats().size(), equalTo(0)); }); @@ -300,6 +297,10 @@ private void assertAllRequestCountsAreZero(RepositoryStatsSnapshot statsSnapshot private List getRepositoriesStats() throws IOException { Map response = getAsMap("/_nodes/_all/_repositories_stats"); + return parseRepositoriesStatsResponse(response); + } + + private List parseRepositoriesStatsResponse(Map response) throws IOException { Map>> nodesRepoStats = extractValue(response, "nodes"); assertThat(response.size(), greaterThan(0)); List repositoriesStats = new ArrayList<>(); @@ -336,7 +337,7 @@ private Set getNodeIds() throws IOException { return nodes.keySet(); } - private void clearRepositoriesStats() throws IOException { + private List clearRepositoriesStats() throws IOException { final Request request = new Request(HttpDelete.METHOD_NAME, "/_nodes/_all/_repositories_stats"); final Response response = client().performRequest(request); assertThat( @@ -344,6 +345,7 @@ private void clearRepositoriesStats() throws IOException { response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()) ); + return parseRepositoriesStatsResponse(responseAsMap(response)); } @SuppressWarnings("unchecked") From b4bb846c0979abb0cdd840f95c6587319af55f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Mon, 10 Aug 2020 18:53:12 +0200 Subject: [PATCH 06/19] Keep track of cluster version on RepositoryStatsSnapshot. Allow clearing up to a particular cluster version. --- .../repositories/RepositoriesService.java | 30 +++++---- .../RepositoriesStatsArchive.java | 63 ++++++++++++------- .../repositories/RepositoryInfo.java | 24 +++---- .../repositories/RepositoryStats.java | 2 +- .../repositories/RepositoryStatsSnapshot.java | 43 +++++++------ .../blobstore/MeteredBlobStoreRepository.java | 9 ++- .../RepositoriesStatsArchiveTests.java | 33 ++++++---- .../ClearRepositoriesStatsArchiveRequest.java | 17 ++++- ...rtClearRepositoriesStatsArchiveAction.java | 18 +++++- ...stClearRepositoriesStatsArchiveAction.java | 5 +- ...tractRepositoriesStatsAPIRestTestCase.java | 16 +++-- .../RepositoriesStatsResponseTests.java | 5 +- 12 files changed, 169 insertions(+), 96 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 74d4434c499f1..74559ec3c2eb4 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -88,6 +88,8 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C private volatile Map repositories = Collections.emptyMap(); private final RepositoriesStatsArchive repositoriesStatsArchive; + private volatile long lastKnownClusterVersion; + public RepositoriesService(Settings settings, ClusterService clusterService, TransportService transportService, Map typesRegistry, Map internalTypesRegistry, ThreadPool threadPool) { @@ -103,8 +105,7 @@ public RepositoriesService(Settings settings, ClusterService clusterService, Tra this.verifyAction = new VerifyNodeRepositoryAction(transportService, clusterService, this); this.repositoriesStatsArchive = new RepositoriesStatsArchive(REPOSITORIES_STATS_ARCHIVE_RETENTION_PERIOD.get(settings), REPOSITORIES_STATS_ARCHIVE_MAX_ARCHIVED_STATS.get(settings), - threadPool::relativeTimeInMillis, - threadPool::absoluteTimeInMillis); + threadPool::relativeTimeInMillis); } /** @@ -139,7 +140,7 @@ public void registerRepository(final PutRepositoryRequest request, final ActionL // Trying to create the new repository on master to make sure it works try { - closeRepository(createRepository(newRepositoryMetadata, typesRegistry), false); + closeRepository(createRepository(newRepositoryMetadata, typesRegistry)); } catch (Exception e) { registrationListener.onFailure(e); return; @@ -306,6 +307,7 @@ protected void doRun() { public void applyClusterState(ClusterChangedEvent event) { try { final ClusterState state = event.state(); + this.lastKnownClusterVersion = state.version(); RepositoriesMetadata oldMetadata = event.previousState().getMetadata().custom(RepositoriesMetadata.TYPE); RepositoriesMetadata newMetadata = state.getMetadata().custom(RepositoriesMetadata.TYPE); @@ -324,7 +326,9 @@ public void applyClusterState(ClusterChangedEvent event) { for (Map.Entry entry : repositories.entrySet()) { if (newMetadata == null || newMetadata.repository(entry.getKey()) == null) { logger.debug("unregistering repository [{}]", entry.getKey()); - closeRepository(entry.getValue()); + Repository repository = entry.getValue(); + closeRepository(repository); + archiveRepositoryStats(repository, state.version()); } else { survivors.put(entry.getKey(), entry.getValue()); } @@ -343,6 +347,7 @@ public void applyClusterState(ClusterChangedEvent event) { // Previous version is different from the version in settings logger.debug("updating repository [{}]", repositoryMetadata.name()); closeRepository(repository); + archiveRepositoryStats(repository, state.version()); repository = null; try { repository = createRepository(repositoryMetadata, typesRegistry); @@ -426,12 +431,12 @@ private List getRepositoryStatsForActiveRepositories() return Stream.concat(repositories.values().stream(), internalRepositories.values().stream()) .filter(r -> r instanceof MeteredBlobStoreRepository) .map(r -> (MeteredBlobStoreRepository) r) - .map(MeteredBlobStoreRepository::statsSnapshot) + .map(r -> r.statsSnapshot(lastKnownClusterVersion)) .collect(Collectors.toList()); } - public List clearRepositoriesStatsArchive() { - return repositoriesStatsArchive.clear(); + public List clearRepositoriesStatsArchive(long maxVersionToClear) { + return repositoriesStatsArchive.clear(maxVersionToClear); } public void registerInternalRepository(String name, String type) { @@ -460,14 +465,13 @@ public void unregisterInternalRepository(String name) { /** Closes the given repository. */ private void closeRepository(Repository repository) { - closeRepository(repository, true); - } - - private void closeRepository(Repository repository, boolean archiveStats) { logger.debug("closing repository [{}][{}]", repository.getMetadata().type(), repository.getMetadata().name()); repository.close(); - if (archiveStats && repository instanceof MeteredBlobStoreRepository) { - RepositoryStatsSnapshot stats = ((MeteredBlobStoreRepository) repository).statsSnapshot(); + } + + private void archiveRepositoryStats(Repository repository, long clusterStateVersion) { + if (repository instanceof MeteredBlobStoreRepository) { + RepositoryStatsSnapshot stats = ((MeteredBlobStoreRepository) repository).statsSnapshotForArchival(clusterStateVersion); if (repositoriesStatsArchive.archive(stats) == false) { logger.warn("Unable to archive the repository stats [{}] as the archive is full.", stats); } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java index fa86d306d5317..68ac085c89e71 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java @@ -24,9 +24,12 @@ import org.elasticsearch.common.unit.TimeValue; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Deque; +import java.util.Iterator; import java.util.List; import java.util.function.LongSupplier; +import java.util.stream.Collectors; public final class RepositoriesStatsArchive { private static final Logger logger = LogManager.getLogger(RepositoriesStatsArchive.class); @@ -34,17 +37,14 @@ public final class RepositoriesStatsArchive { private final TimeValue retentionPeriod; private final int maxCapacity; private final LongSupplier relativeTimeSupplier; - private final LongSupplier absoluteTimeSupplier; - private final Deque archive = new ArrayDeque<>(); + private final Deque archive = new ArrayDeque<>(); public RepositoriesStatsArchive(TimeValue retentionPeriod, int maxCapacity, - LongSupplier relativeTimeSupplier, - LongSupplier absoluteTimeSupplier) { + LongSupplier relativeTimeSupplier) { this.retentionPeriod = retentionPeriod; this.maxCapacity = maxCapacity; this.relativeTimeSupplier = relativeTimeSupplier; - this.absoluteTimeSupplier = absoluteTimeSupplier; } /** @@ -56,22 +56,20 @@ public RepositoriesStatsArchive(TimeValue retentionPeriod, synchronized boolean archive(final RepositoryStatsSnapshot repositoryStats) { assert containsRepositoryStats(repositoryStats) == false : "A repository with ephemeral id " + repositoryStats.getRepositoryInfo().ephemeralId + " is already archived"; + assert repositoryStats.isArchived(); + evict(); if (archive.size() >= maxCapacity) { return false; } - RepositoryInfo stoppedRepoInfo = - repositoryStats.getRepositoryInfo().stopped(absoluteTimeSupplier.getAsLong()); - RepositoryStatsSnapshot stoppedStats = - new RepositoryStatsSnapshot(stoppedRepoInfo, repositoryStats.getRepositoryStats(), relativeTimeSupplier.getAsLong()); - return archive.add(stoppedStats); + return archive.add(new ArchiveEntry(repositoryStats, relativeTimeSupplier.getAsLong())); } synchronized List getArchivedStats() { evict(); - return List.copyOf(archive); + return archive.stream().map(e -> e.repositoryStatsSnapshot).collect(Collectors.toList()); } /** @@ -79,22 +77,45 @@ synchronized List getArchivedStats() { * * @return the repository stats that were stored before clearing the archive. */ - synchronized List clear() { - List archivedStats = getArchivedStats(); - archive.clear(); - logger.debug("RepositoriesStatsArchive have been cleared. Removed stats: [{}]", archivedStats); - return archivedStats; + synchronized List clear(long maxVersionToClear) { + List clearedStats = new ArrayList<>(); + Iterator iterator = archive.iterator(); + while (iterator.hasNext()) { + RepositoryStatsSnapshot statsSnapshot = iterator.next().repositoryStatsSnapshot; + if (statsSnapshot.getClusterVersion() <= maxVersionToClear) { + clearedStats.add(statsSnapshot); + iterator.remove(); + } + } + logger.debug("RepositoriesStatsArchive have been cleared. Removed stats: [{}]", clearedStats); + return clearedStats; } private void evict() { - RepositoryStatsSnapshot stats; - while ((stats = archive.peek()) != null && stats.ageInMillis(relativeTimeSupplier) >= retentionPeriod.getMillis()) { - RepositoryStatsSnapshot removedStats = archive.poll(); - logger.debug("Evicting repository stats [{}]", removedStats); + ArchiveEntry entry; + while ((entry = archive.peek()) != null && entry.ageInMillis(relativeTimeSupplier) >= retentionPeriod.getMillis()) { + ArchiveEntry removedEntry = archive.poll(); + logger.debug("Evicting repository stats [{}]", removedEntry); } } private boolean containsRepositoryStats(RepositoryStatsSnapshot repositoryStats) { - return archive.stream().anyMatch(r -> r.getRepositoryInfo().ephemeralId.equals(repositoryStats.getRepositoryInfo().ephemeralId)); + return archive.stream() + .anyMatch(entry -> + entry.repositoryStatsSnapshot.getRepositoryInfo().ephemeralId.equals(repositoryStats.getRepositoryInfo().ephemeralId)); + } + + private static class ArchiveEntry { + private final RepositoryStatsSnapshot repositoryStatsSnapshot; + private final long createdAtMillis; + + private ArchiveEntry(RepositoryStatsSnapshot repositoryStatsSnapshot, long createdAtMillis) { + this.repositoryStatsSnapshot = repositoryStatsSnapshot; + this.createdAtMillis = createdAtMillis; + } + + private long ageInMillis(LongSupplier relativeTimeInMillis) { + return Math.max(0, relativeTimeInMillis.getAsLong() - createdAtMillis); + } } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java index c41f5bfe74d2f..2361767dcad1b 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java @@ -20,6 +20,7 @@ package org.elasticsearch.repositories; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -42,7 +43,7 @@ public RepositoryInfo(String ephemeralId, String name, String type, String location, - Long startedAt) { + long startedAt) { this(ephemeralId, name, type, location, startedAt, null); } @@ -50,7 +51,7 @@ public RepositoryInfo(String ephemeralId, String name, String type, String location, - Long startedAt, + long startedAt, @Nullable Long stoppedAt) { this.ephemeralId = ephemeralId; this.name = name; @@ -110,11 +111,11 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RepositoryInfo that = (RepositoryInfo) o; - return Objects.equals(ephemeralId, that.ephemeralId) && - Objects.equals(name, that.name) && - Objects.equals(type, that.type) && - Objects.equals(location, that.location) && - Objects.equals(startedAt, that.startedAt) && + return ephemeralId.equals(that.ephemeralId) && + name.equals(that.name) && + type.equals(that.type) && + location.equals(that.location) && + startedAt == that.startedAt && Objects.equals(stoppedAt, that.stoppedAt); } @@ -125,13 +126,6 @@ public int hashCode() { @Override public String toString() { - return "RepositoryInfo{" + - "ephemeralId='" + ephemeralId + '\'' + - ", name='" + name + '\'' + - ", type='" + type + '\'' + - ", location='" + location + '\'' + - ", startedAt=" + startedAt + - ", stoppedAt=" + stoppedAt + - '}'; + return Strings.toString(this); } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java index e4fe80510a549..d6fc680946b38 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryStats.java @@ -62,7 +62,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RepositoryStats that = (RepositoryStats) o; - return Objects.equals(requestCounts, that.requestCounts); + return requestCounts.equals(that.requestCounts); } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java index 809ac3ddbef7a..3b805d01d31ca 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java @@ -19,7 +19,7 @@ package org.elasticsearch.repositories; -import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -28,26 +28,28 @@ import java.io.IOException; import java.util.Objects; -import java.util.function.LongSupplier; public final class RepositoryStatsSnapshot implements Writeable, ToXContent { private final RepositoryInfo repositoryInfo; private final RepositoryStats repositoryStats; - @Nullable - private final Long createdAtMillis; + private final long clusterVersion; + private final boolean archived; public RepositoryStatsSnapshot(RepositoryInfo repositoryInfo, RepositoryStats repositoryStats, - @Nullable Long createdAtMillis) { + long clusterVersion, + boolean archived) { this.repositoryInfo = repositoryInfo; this.repositoryStats = repositoryStats; - this.createdAtMillis = createdAtMillis; + this.clusterVersion = clusterVersion; + this.archived = archived; } public RepositoryStatsSnapshot(StreamInput in) throws IOException { this.repositoryInfo = new RepositoryInfo(in); this.repositoryStats = new RepositoryStats(in); - this.createdAtMillis = null; + this.clusterVersion = in.readLong(); + this.archived = in.readBoolean(); } public RepositoryInfo getRepositoryInfo() { @@ -58,21 +60,29 @@ public RepositoryStats getRepositoryStats() { return repositoryStats; } - public long ageInMillis(LongSupplier relativeTimeInMillis) { - return createdAtMillis == null ? 0 : relativeTimeInMillis.getAsLong() - createdAtMillis; + public boolean isArchived() { + return archived; + } + + public long getClusterVersion() { + return clusterVersion; } @Override public void writeTo(StreamOutput out) throws IOException { repositoryInfo.writeTo(out); repositoryStats.writeTo(out); + out.writeLong(clusterVersion); + out.writeBoolean(archived); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); repositoryInfo.toXContent(builder, params); + builder.field("cluster_version", clusterVersion); builder.field("request_counts", repositoryStats.requestCounts); + builder.field("archived", archived); builder.endObject(); return builder; } @@ -82,22 +92,19 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RepositoryStatsSnapshot that = (RepositoryStatsSnapshot) o; - return Objects.equals(repositoryInfo, that.repositoryInfo) && - Objects.equals(repositoryStats, that.repositoryStats) && - Objects.equals(createdAtMillis, that.createdAtMillis); + return repositoryInfo.equals(that.repositoryInfo) && + repositoryStats.equals(that.repositoryStats) && + clusterVersion == that.clusterVersion && + archived == that.archived; } @Override public int hashCode() { - return Objects.hash(repositoryInfo, repositoryStats, createdAtMillis); + return Objects.hash(repositoryInfo, repositoryStats, clusterVersion, archived); } @Override public String toString() { - return "RepositoryStatsSnapshot{" + - "repositoryInfo=" + repositoryInfo + - ", repositoryStats=" + repositoryStats + - ", createdAtMillis=" + createdAtMillis + - '}'; + return Strings.toString(this); } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java index b91c76dac4dd0..2a1ae0e525f98 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java @@ -47,8 +47,13 @@ public MeteredBlobStoreRepository(RepositoryMetadata metadata, threadPool.absoluteTimeInMillis()); } - public RepositoryStatsSnapshot statsSnapshot() { - return new RepositoryStatsSnapshot(repositoryInfo, stats(), threadPool.relativeTimeInMillis()); + public RepositoryStatsSnapshot statsSnapshot(long clusterVersion) { + return new RepositoryStatsSnapshot(repositoryInfo, stats(), clusterVersion, false); + } + + public RepositoryStatsSnapshot statsSnapshotForArchival(long clusterVersion) { + RepositoryInfo stoppedRepoInfo = repositoryInfo.stopped(threadPool.absoluteTimeInMillis()); + return new RepositoryStatsSnapshot(stoppedRepoInfo, stats(), clusterVersion, true); } private static String getLocation(BlobPath basePath, String bucket) { diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java index 46ed159352c0c..a17b99f2b8b7f 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java @@ -37,8 +37,7 @@ public void testStatsAreEvictedOnceTheyAreOlderThanRetentionPeriod() { RepositoriesStatsArchive repositoriesStatsArchive = new RepositoriesStatsArchive(TimeValue.timeValueMillis(retentionTimeInMillis), 100, - fakeRelativeClock::get, - System::currentTimeMillis); + fakeRelativeClock::get); for (int i = 0; i < randomInt(10); i++) { RepositoryStatsSnapshot repoStats = createRepositoryStats(RepositoryStats.EMPTY_STATS); @@ -66,8 +65,7 @@ public void testStatsAreRejectedIfTheArchiveIsFull() { RepositoriesStatsArchive repositoriesStatsArchive = new RepositoriesStatsArchive(TimeValue.timeValueMillis(retentionTimeInMillis), 1, - fakeRelativeClock::get, - System::currentTimeMillis); + fakeRelativeClock::get); assertTrue(repositoriesStatsArchive.archive(createRepositoryStats(RepositoryStats.EMPTY_STATS))); @@ -84,27 +82,36 @@ public void testClearArchive() { RepositoriesStatsArchive repositoriesStatsArchive = new RepositoriesStatsArchive(TimeValue.timeValueMillis(retentionTimeInMillis), 100, - fakeRelativeClock::get, - System::currentTimeMillis); + fakeRelativeClock::get); - int archivedStats = randomInt(20); - for (int i = 0; i < archivedStats; i++) { - repositoriesStatsArchive.archive(createRepositoryStats(RepositoryStats.EMPTY_STATS)); + int archivedStatsWithVersionZero = randomIntBetween(1, 20); + for (int i = 0; i < archivedStatsWithVersionZero; i++) { + repositoriesStatsArchive.archive(createRepositoryStats(RepositoryStats.EMPTY_STATS, 0)); } - List removedStats = repositoriesStatsArchive.clear(); - assertThat(removedStats.size(), equalTo(archivedStats)); + int archivedStatsWithNewerVersion = randomIntBetween(1, 20); + for (int i = 0; i < archivedStatsWithNewerVersion; i++) { + repositoriesStatsArchive.archive(createRepositoryStats(RepositoryStats.EMPTY_STATS, 1)); + } + + List removedStats = repositoriesStatsArchive.clear(0L); + assertThat(removedStats.size(), equalTo(archivedStatsWithVersionZero)); - assertThat(repositoriesStatsArchive.getArchivedStats().size(), equalTo(0)); + assertThat(repositoriesStatsArchive.getArchivedStats().size(), equalTo(archivedStatsWithNewerVersion)); } private RepositoryStatsSnapshot createRepositoryStats(RepositoryStats repositoryStats) { + return createRepositoryStats(repositoryStats, 0); + } + + private RepositoryStatsSnapshot createRepositoryStats(RepositoryStats repositoryStats, long clusterVersion) { RepositoryInfo repositoryInfo = new RepositoryInfo(UUIDs.randomBase64UUID(), randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), System.currentTimeMillis(), null); - return new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, null); + return new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, clusterVersion, true); } + } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java index ec0fbbc515448..ab9e414fed2f9 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java @@ -8,15 +8,30 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; public final class ClearRepositoriesStatsArchiveRequest extends BaseNodesRequest { + private final long maxVersionToClear; + public ClearRepositoriesStatsArchiveRequest(StreamInput in) throws IOException { super(in); + this.maxVersionToClear = in.readLong(); } - public ClearRepositoriesStatsArchiveRequest(String... nodesIds) { + public ClearRepositoriesStatsArchiveRequest(long maxVersionToClear, String... nodesIds) { super(nodesIds); + this.maxVersionToClear = maxVersionToClear; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(maxVersionToClear); + } + + public long getMaxVersionToClear() { + return maxVersionToClear; } } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java index 0edbbfffb430a..76e628f15bf6b 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.RepositoryStatsSnapshot; import org.elasticsearch.tasks.Task; @@ -63,7 +64,7 @@ protected RepositoriesStatsResponse newResponse( @Override protected ClearRepositoriesStatsArchiveNodeRequest newNodeRequest(ClearRepositoriesStatsArchiveRequest request) { - return new ClearRepositoriesStatsArchiveNodeRequest(); + return new ClearRepositoriesStatsArchiveNodeRequest(request.getMaxVersionToClear()); } @Override @@ -73,15 +74,26 @@ protected RepositoriesNodeStatsResponse newNodeResponse(StreamInput in) throws I @Override protected RepositoriesNodeStatsResponse nodeOperation(ClearRepositoriesStatsArchiveNodeRequest request, Task task) { - List clearedStats = repositoriesService.clearRepositoriesStatsArchive(); + List clearedStats = repositoriesService.clearRepositoriesStatsArchive(request.maxVersionToClear); return new RepositoriesNodeStatsResponse(clusterService.localNode(), clearedStats); } static final class ClearRepositoriesStatsArchiveNodeRequest extends TransportRequest { - ClearRepositoriesStatsArchiveNodeRequest() {} + private final long maxVersionToClear; + + public ClearRepositoriesStatsArchiveNodeRequest(long maxVersionToClear) { + this.maxVersionToClear = maxVersionToClear; + } ClearRepositoriesStatsArchiveNodeRequest(StreamInput in) throws IOException { super(in); + this.maxVersionToClear = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(maxVersionToClear); } } } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java index 2bf3977c32a89..658ca16de9999 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java @@ -24,13 +24,14 @@ public String getName() { @Override public List routes() { - return List.of(new Route(RestRequest.Method.DELETE, "/_nodes/{nodeId}/_repositories_stats")); + return List.of(new Route(RestRequest.Method.DELETE, "/_nodes/{nodeId}/_repositories_stats/{maxVersionToClear}")); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); - ClearRepositoriesStatsArchiveRequest clearArchivesRequest = new ClearRepositoriesStatsArchiveRequest(nodesIds); + long maxVersionToClear = request.paramAsLong("maxVersionToClear", -1); + ClearRepositoriesStatsArchiveRequest clearArchivesRequest = new ClearRepositoriesStatsArchiveRequest(maxVersionToClear, nodesIds); return channel -> client.execute( ClearRepositoriesStatsArchiveAction.INSTANCE, clearArchivesRequest, diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java index 67631d2840217..c0de6b400fd3c 100644 --- a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java +++ b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java @@ -55,7 +55,7 @@ public abstract class AbstractRepositoriesStatsAPIRestTestCase extends ESRestTes @Before public void clearArchive() throws Exception { - clearRepositoriesStats(); + clearRepositoriesStats(Long.MAX_VALUE); } public void testStatsAreTracked() throws Exception { @@ -106,8 +106,11 @@ public void testClearRepositoriesStats() throws Exception { List repositoriesStatsBeforeClearing = getRepositoriesStats(); assertThat(repositoriesStatsBeforeClearing.size(), equalTo(1)); + RepositoryStatsSnapshot repositoryStatsSnapshot = repositoriesStatsBeforeClearing.get(0); - List removedRepositoriesStats = clearRepositoriesStats(); + assertThat(clearRepositoriesStats(-1).size(), equalTo(0)); + + List removedRepositoriesStats = clearRepositoriesStats(repositoryStatsSnapshot.getClusterVersion()); assertThat(repositoriesStatsBeforeClearing, equalTo(removedRepositoriesStats)); @@ -311,11 +314,14 @@ private List parseRepositoriesStatsResponse(Map nodeStatSnapshot : nodeStats) { RepositoryInfo repositoryInfo = parseRepositoryInfo(nodeStatSnapshot); Map intRequestCounters = extractValue(nodeStatSnapshot, "request_counts"); + int clusterVersion = extractValue(nodeStatSnapshot, "cluster_version"); + boolean archived = extractValue(nodeStatSnapshot, "archived"); Map requestCounters = intRequestCounters.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().longValue())); RepositoryStats repositoryStats = new RepositoryStats(requestCounters); - RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, null); + RepositoryStatsSnapshot statsSnapshot = + new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, clusterVersion, archived); repositoriesStats.add(statsSnapshot); } } @@ -337,8 +343,8 @@ private Set getNodeIds() throws IOException { return nodes.keySet(); } - private List clearRepositoriesStats() throws IOException { - final Request request = new Request(HttpDelete.METHOD_NAME, "/_nodes/_all/_repositories_stats"); + private List clearRepositoriesStats(long maxVersionToClear) throws IOException { + final Request request = new Request(HttpDelete.METHOD_NAME, "/_nodes/_all/_repositories_stats/" + maxVersionToClear); final Response response = client().performRequest(request); assertThat( "Failed to clear repositories stats: " + response, diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java index 689e654322fd3..8637c9f64c9f5 100644 --- a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java +++ b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java @@ -69,13 +69,14 @@ private RepositoriesStatsResponse createResponse() { String repoName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); String repoType = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); String repoLocation = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); - Long startedAt = System.currentTimeMillis() - 1; + long startedAt = System.currentTimeMillis() - 1; Long stoppedAt = randomBoolean() ? System.currentTimeMillis() : null; RepositoryInfo repositoryInfo = new RepositoryInfo(repoId, repoName, repoType, repoLocation, startedAt, stoppedAt); RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot( repositoryInfo, new RepositoryStats(Map.of("GET", randomLongBetween(0, 2000))), - null + j, + randomBoolean() ); nodeRepoStats.add(statsSnapshot); } From d0a5995a63e5d617c0a2ee1a7512a84c3c26df60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 11 Aug 2020 10:25:37 +0200 Subject: [PATCH 07/19] Fix test --- .../RepositoriesServiceTests.java | 29 ++++++++++++++++--- ...rtClearRepositoriesStatsArchiveAction.java | 2 +- ...tractRepositoriesStatsAPIRestTestCase.java | 8 +++-- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index be2456e3b0d3f..5e022f7605ae3 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -23,10 +23,13 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterApplierService; @@ -85,7 +88,7 @@ public void setUp() throws Exception { MeteredRepositoryTypeA.TYPE, metadata -> new MeteredRepositoryTypeA(metadata, clusterService), MeteredRepositoryTypeB.TYPE, metadata -> new MeteredRepositoryTypeB(metadata, clusterService)); repositoriesService = new RepositoriesService(Settings.EMPTY, mock(ClusterService.class), - transportService, Collections.emptyMap(), typesRegistry, threadPool); + transportService, typesRegistry, typesRegistry, threadPool); repositoriesService.start(); } @@ -134,13 +137,17 @@ public void testRegisterRejectsInvalidRepositoryNames() { public void testRepositoriesStatsCanHaveTheSameNameAndDifferentTypeOverTime() { String repoName = "name"; expectThrows(RepositoryMissingException.class, () -> repositoriesService.repository(repoName)); - repositoriesService.registerInternalRepository(repoName, MeteredRepositoryTypeA.TYPE); + + ClusterState clusterStateWithRepoTypeA = createClusterStateWithRepo(repoName, MeteredRepositoryTypeA.TYPE); + + repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeA, emptyState())); assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); - repositoriesService.unregisterInternalRepository(repoName); + repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", emptyState(), clusterStateWithRepoTypeA)); assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); - repositoriesService.registerInternalRepository(repoName, MeteredRepositoryTypeB.TYPE); + ClusterState clusterStateWithRepoTypeB = createClusterStateWithRepo(repoName, MeteredRepositoryTypeB.TYPE); + repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeB, emptyState())); List repositoriesStats = repositoriesService.repositoriesStats(); assertThat(repositoriesStats.size(), equalTo(2)); @@ -153,6 +160,20 @@ public void testRepositoriesStatsCanHaveTheSameNameAndDifferentTypeOverTime() { assertThat(repositoryStatsTypeB.getRepositoryStats(), equalTo(MeteredRepositoryTypeB.STATS)); } + private ClusterState createClusterStateWithRepo(String repoName, String repoType) { + ClusterState.Builder state = ClusterState.builder(new ClusterName("test")); + Metadata.Builder mdBuilder = Metadata.builder(); + mdBuilder.putCustom(RepositoriesMetadata.TYPE, + new RepositoriesMetadata(Collections.singletonList(new RepositoryMetadata(repoName, repoType, Settings.EMPTY)))); + state.metadata(mdBuilder); + + return state.build(); + } + + private ClusterState emptyState() { + return ClusterState.builder(new ClusterName("test")).build(); + } + private void assertThrowsOnRegister(String repoName) { PutRepositoryRequest request = new PutRepositoryRequest(repoName); expectThrows(RepositoryException.class, () -> repositoriesService.registerRepository(request, null)); diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java index 76e628f15bf6b..4ec883253ba1e 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java @@ -81,7 +81,7 @@ protected RepositoriesNodeStatsResponse nodeOperation(ClearRepositoriesStatsArch static final class ClearRepositoriesStatsArchiveNodeRequest extends TransportRequest { private final long maxVersionToClear; - public ClearRepositoriesStatsArchiveNodeRequest(long maxVersionToClear) { + ClearRepositoriesStatsArchiveNodeRequest(long maxVersionToClear) { this.maxVersionToClear = maxVersionToClear; } diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java index c0de6b400fd3c..db17bfc719b44 100644 --- a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java +++ b/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java @@ -320,8 +320,12 @@ private List parseRepositoriesStatsResponse(Map e.getValue().longValue())); RepositoryStats repositoryStats = new RepositoryStats(requestCounters); - RepositoryStatsSnapshot statsSnapshot = - new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, clusterVersion, archived); + RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot( + repositoryInfo, + repositoryStats, + clusterVersion, + archived + ); repositoriesStats.add(statsSnapshot); } } From d8aea38b0403d0ce0b99485ede743c5fb7d00461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 11 Aug 2020 14:19:52 +0200 Subject: [PATCH 08/19] Rename repositories-stats to repositories-metering --- ...lear-repositories-metrics-archive.asciidoc | 37 +++++++++++++++++++ .../apis/get-repositories-metrics.asciidoc} | 36 +++++++++++------- .../repositories-metering.asciidoc | 15 ++++++++ .../clear-repositories-stats-archive.asciidoc | 31 ---------------- .../apis/repositories-stats.asciidoc | 12 ------ .../fixtures/azure-fixture/docker-compose.yml | 2 +- test/fixtures/gcs-fixture/docker-compose.yml | 2 +- test/fixtures/s3-fixture/docker-compose.yml | 2 +- .../build.gradle | 8 ++-- .../qa/azure/build.gradle | 18 ++++----- .../azure/AzureRepositoriesMetricsIT.java} | 10 ++--- .../qa/build.gradle | 0 .../qa/gcs/build.gradle | 14 +++---- .../gcs/GCSRepositoriesMetricsIT.java} | 10 ++--- .../qa/s3/build.gradle | 8 ++-- .../metrics/s3/S3RepositoriesMetricsIT.java} | 6 +-- .../metrics/RepositoriesMeteringPlugin.java} | 23 ++++++------ ...ClearRepositoriesMetricsArchiveAction.java | 19 ++++++++++ ...earRepositoriesMetricsArchiveRequest.java} | 8 ++-- .../action/RepositoriesMetricsAction.java | 19 ++++++++++ .../action/RepositoriesMetricsRequest.java} | 8 ++-- .../action/RepositoriesMetricsResponse.java} | 18 ++++----- .../RepositoriesNodeMetricsResponse.java} | 8 ++-- ...rtClearRepositoriesStatsArchiveAction.java | 32 ++++++++-------- .../TransportRepositoriesStatsAction.java | 32 ++++++++-------- ...learRepositoriesMetricsArchiveAction.java} | 16 ++++---- .../RestGetRepositoriesMetricsAction.java} | 18 ++++----- ...ctRepositoriesMetricsAPIRestTestCase.java} | 8 ++-- .../RepositoriesMetricsResponseTests.java} | 32 ++++++++-------- .../ClearRepositoriesStatsArchiveAction.java | 19 ---------- .../stats/action/RepositoriesStatsAction.java | 19 ---------- 31 files changed, 254 insertions(+), 236 deletions(-) create mode 100644 docs/reference/repositories-metering/apis/clear-repositories-metrics-archive.asciidoc rename docs/reference/{repositories-stats/apis/get-repositories-stats.asciidoc => repositories-metering/apis/get-repositories-metrics.asciidoc} (78%) create mode 100644 docs/reference/repositories-metering/repositories-metering.asciidoc delete mode 100644 docs/reference/repositories-stats/apis/clear-repositories-stats-archive.asciidoc delete mode 100644 docs/reference/repositories-stats/apis/repositories-stats.asciidoc rename x-pack/plugin/{repositories-stats => repositories-metering-api}/build.gradle (82%) rename x-pack/plugin/{repositories-stats => repositories-metering-api}/qa/azure/build.gradle (83%) rename x-pack/plugin/{repositories-stats/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/stats/azure/AzureRepositoriesStatsIT.java => repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metrics/azure/AzureRepositoriesMetricsIT.java} (71%) rename x-pack/plugin/{repositories-stats => repositories-metering-api}/qa/build.gradle (100%) rename x-pack/plugin/{repositories-stats => repositories-metering-api}/qa/gcs/build.gradle (87%) rename x-pack/plugin/{repositories-stats/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/stats/gcs/GCSRepositoriesStatsIT.java => repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metrics/gcs/GCSRepositoriesMetricsIT.java} (71%) rename x-pack/plugin/{repositories-stats => repositories-metering-api}/qa/s3/build.gradle (90%) rename x-pack/plugin/{repositories-stats/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/stats/s3/S3RepositoriesStatsIT.java => repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metrics/s3/S3RepositoriesMetricsIT.java} (84%) rename x-pack/plugin/{repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/RepositoriesStatsPlugin.java => repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/RepositoriesMeteringPlugin.java} (57%) create mode 100644 x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveAction.java rename x-pack/plugin/{repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java => repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveRequest.java} (70%) create mode 100644 x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsAction.java rename x-pack/plugin/{repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsRequest.java => repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsRequest.java} (60%) rename x-pack/plugin/{repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponse.java => repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponse.java} (65%) rename x-pack/plugin/{repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsResponse.java => repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesNodeMetricsResponse.java} (80%) rename x-pack/plugin/{repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats => repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics}/action/TransportClearRepositoriesStatsArchiveAction.java (73%) rename x-pack/plugin/{repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats => repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics}/action/TransportRepositoriesStatsAction.java (67%) rename x-pack/plugin/{repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java => repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java} (62%) rename x-pack/plugin/{repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestGetRepositoriesStatsAction.java => repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestGetRepositoriesMetricsAction.java} (62%) rename x-pack/plugin/{repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java => repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/AbstractRepositoriesMetricsAPIRestTestCase.java} (98%) rename x-pack/plugin/{repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java => repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponseTests.java} (74%) delete mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java delete mode 100644 x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsAction.java diff --git a/docs/reference/repositories-metering/apis/clear-repositories-metrics-archive.asciidoc b/docs/reference/repositories-metering/apis/clear-repositories-metrics-archive.asciidoc new file mode 100644 index 0000000000000..c5875a4591d97 --- /dev/null +++ b/docs/reference/repositories-metering/apis/clear-repositories-metrics-archive.asciidoc @@ -0,0 +1,37 @@ +[[clear-repositories-metrics-archive-api]] +=== Clear repositories metrics archive API +++++ +Clear repositories metrics archive API +++++ + +Removes the archived repositories metrics in the cluster up to +a particular version. + +[[clear-repositories-metrics-archive-api-request]] +==== {api-request-title} + +`DELETE /_nodes//_repositories_metrics/` + +[[clear-repositories-metrics-archive-api-desc]] +==== {api-description-title} + +You can clear the archived repositories utilization statistics using this API +up to a version. + +All the nodes selective options are explained <>. + +[[clear-repositories-metrics-archive-api-path-params]] +==== {api-path-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] + +``:: + (long) Specify the max version to be cleared from the archive. + +[[clear-repositories-metrics-archive-api-query-params]] +==== {api-query-parms-title} + +[role="child_attributes"] +[[clear-repositories-metrics-archive-api-response-body]] +==== {api-response-body-title} +It returns all the deleted repositories metrics. diff --git a/docs/reference/repositories-stats/apis/get-repositories-stats.asciidoc b/docs/reference/repositories-metering/apis/get-repositories-metrics.asciidoc similarity index 78% rename from docs/reference/repositories-stats/apis/get-repositories-stats.asciidoc rename to docs/reference/repositories-metering/apis/get-repositories-metrics.asciidoc index 9ee5b8eef5953..dd1aa2aae5cde 100644 --- a/docs/reference/repositories-stats/apis/get-repositories-stats.asciidoc +++ b/docs/reference/repositories-metering/apis/get-repositories-metrics.asciidoc @@ -1,34 +1,34 @@ -[[get-repositories-stats-api]] -=== Repositories statistics API +[[get-repositories-metrics-api]] +=== Repositories metrics API ++++ -Repositories statistics +Repositories metrics API ++++ -Returns cluster repositories utilization statistics. +Returns cluster repositories metrics. -[[get-repositories-stats-api-request]] +[[get-repositories-metrics-api-request]] ==== {api-request-title} -`GET /_nodes//_repositories_stats` +`GET /_nodes//_repositories_metrics` -[[get-repositories-stats-api-desc]] +[[get-repositories-metrics-api-desc]] ==== {api-description-title} -You can use the cluster repositories statistics API to retrieve repository utilization statistics in a cluster. +You can use the cluster repositories metrics API to retrieve repositories metrics in a cluster. All the nodes selective options are explained <>. -[[get-repositories-stats-api-path-params]] +[[get-repositories-metrics-api-path-params]] ==== {api-path-parms-title} include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] -[[get-repositories-stats-api-query-params]] +[[get-repositories-metrics-api-query-params]] ==== {api-query-parms-title} [role="child_attributes"] -[[get-repositories-stats-api-response-body]] +[[get-repositories-metrics-api-response-body]] ==== {api-response-body-title} `_nodes`:: @@ -58,14 +58,14 @@ Name of the cluster. Based on the <> setting. `nodes`:: (object) -Contains repository utilization statistics for the nodes selected by the request. +Contains repositories metrics for the nodes selected by the request. + .Properties of `nodes` [%collapsible%open] ==== ``:: (array) -An array of repository utilization statistics for the node. +An array of repository metrics for the node. + .Properties of objects in `node_id` [%collapsible%open] @@ -97,6 +97,16 @@ since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. Time the repository was deleted or updated. Recorded in milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. +`archived`:: +(boolean) +A flag that tells whether or not this object has been archived. + +`cluster_version`:: +(long) +The cluster state version when this object was created, this field +can be used as a logical timestamp to delete all the archived metrics up +to a observed cluster version. + `request_counts`:: (object) An object with the number of request performed against the repository diff --git a/docs/reference/repositories-metering/repositories-metering.asciidoc b/docs/reference/repositories-metering/repositories-metering.asciidoc new file mode 100644 index 0000000000000..52bdc53480762 --- /dev/null +++ b/docs/reference/repositories-metering/repositories-metering.asciidoc @@ -0,0 +1,15 @@ +[role="xpack"] +[testenv="enterprise"] +[[repositories-metering-apis]] +== Repositories metering APIs + +experimental[] + +You can use the following APIs to retrieve repositories metrics. +This is an API meant to be used by Elastic's commercial offerings. + +* <> +* <> + +include::get-repositories-metrics.asciidoc[] +include::clear-repositories-metrics-archive.asciidoc[] diff --git a/docs/reference/repositories-stats/apis/clear-repositories-stats-archive.asciidoc b/docs/reference/repositories-stats/apis/clear-repositories-stats-archive.asciidoc deleted file mode 100644 index d7e25bfbfb730..0000000000000 --- a/docs/reference/repositories-stats/apis/clear-repositories-stats-archive.asciidoc +++ /dev/null @@ -1,31 +0,0 @@ -[[clear-repositories-stats-archive-api]] -=== Clear repositories statistics archive API -++++ -Clear repositories statistics archive API -++++ - -Removes the archived repositories statistics in the cluster. - -[[clear-repositories-stats-archive-api-request]] -==== {api-request-title} - -`DELETE /_nodes//_repositories_stats` - -[[clear-repositories-stats-archive-api-desc]] -==== {api-description-title} - -You can clear the archived repositories utilization statistics using this API. - -All the nodes selective options are explained <>. - -[[clear-repositories-stats-archive-api-path-params]] -==== {api-path-parms-title} - -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] - -[[clear-repositories-stats-archive-api-query-params]] -==== {api-query-parms-title} - -[role="child_attributes"] -[[clear-repositories-stats-archive-api-response-body]] -==== {api-response-body-title} diff --git a/docs/reference/repositories-stats/apis/repositories-stats.asciidoc b/docs/reference/repositories-stats/apis/repositories-stats.asciidoc deleted file mode 100644 index 2ebb0d9da2c89..0000000000000 --- a/docs/reference/repositories-stats/apis/repositories-stats.asciidoc +++ /dev/null @@ -1,12 +0,0 @@ -[[repositories-stats-apis]] -== Repositories utilization statistics APIs - -experimental[] - -You can use the following APIs to retrieve statistics about the repositories utilization. - -* <> -* <> - -include::get-repositories-stats.asciidoc[] -include::clear-repositories-stats-archive.asciidoc[] diff --git a/test/fixtures/azure-fixture/docker-compose.yml b/test/fixtures/azure-fixture/docker-compose.yml index 782ba8127dc89..85e073e1803c1 100644 --- a/test/fixtures/azure-fixture/docker-compose.yml +++ b/test/fixtures/azure-fixture/docker-compose.yml @@ -18,7 +18,7 @@ services: ports: - "8091" - azure-fixture-repositories-stats: + azure-fixture-repositories-metering: build: context: . dockerfile: Dockerfile diff --git a/test/fixtures/gcs-fixture/docker-compose.yml b/test/fixtures/gcs-fixture/docker-compose.yml index 2c8cba84a01d4..30a362e7caa8d 100644 --- a/test/fixtures/gcs-fixture/docker-compose.yml +++ b/test/fixtures/gcs-fixture/docker-compose.yml @@ -36,7 +36,7 @@ services: - ./testfixtures_shared/shared:/fixture/shared ports: - "80" - gcs-fixture-repositories-stats: + gcs-fixture-repositories-metering: build: context: . args: diff --git a/test/fixtures/s3-fixture/docker-compose.yml b/test/fixtures/s3-fixture/docker-compose.yml index 70d494b5990d4..22d101f41c318 100644 --- a/test/fixtures/s3-fixture/docker-compose.yml +++ b/test/fixtures/s3-fixture/docker-compose.yml @@ -30,7 +30,7 @@ services: ports: - "80" - s3-fixture-repositories-stats: + s3-fixture-repositories-metering: build: context: . args: diff --git a/x-pack/plugin/repositories-stats/build.gradle b/x-pack/plugin/repositories-metering-api/build.gradle similarity index 82% rename from x-pack/plugin/repositories-stats/build.gradle rename to x-pack/plugin/repositories-metering-api/build.gradle index 2784fe02dd5c1..998128d168a3b 100644 --- a/x-pack/plugin/repositories-stats/build.gradle +++ b/x-pack/plugin/repositories-metering-api/build.gradle @@ -2,12 +2,12 @@ evaluationDependsOn(xpackModule('core')) apply plugin: 'elasticsearch.esplugin' esplugin { - name 'repositories-stats' - description 'Repositories stats API' - classname 'org.elasticsearch.xpack.repositories.stats.RepositoriesStatsPlugin' + name 'repositories-metering-api' + description 'Repositories metering API' + classname 'org.elasticsearch.xpack.repositories.metrics.RepositoriesMeteringPlugin' extendedPlugins = ['x-pack-core'] } -archivesBaseName = 'x-pack-repositories-stats' +archivesBaseName = 'x-pack-repositories-metering-api' dependencies { compileOnly project(path: xpackModule('core'), configuration: 'default') diff --git a/x-pack/plugin/repositories-stats/qa/azure/build.gradle b/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle similarity index 83% rename from x-pack/plugin/repositories-stats/qa/azure/build.gradle rename to x-pack/plugin/repositories-metering-api/qa/azure/build.gradle index c9d53346e66c8..560d6fad19364 100644 --- a/x-pack/plugin/repositories-stats/qa/azure/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle @@ -27,14 +27,14 @@ final Project fixture = project(':test:fixtures:azure-fixture') final Project repositoryPlugin = project(':plugins:repository-azure') dependencies { - testImplementation project(path: xpackModule('repositories-stats'), configuration: 'testArtifacts') + testImplementation project(path: xpackModule('repositories-metering-api'), configuration: 'testArtifacts') testImplementation repositoryPlugin } restResources { restApi { includeCore 'indices', 'bulk', 'snapshot', 'nodes', '_common' - includeXpack 'repositories-stats' + includeXpack 'repositories-metering-api' } } @@ -57,25 +57,25 @@ if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSas if (useFixture) { apply plugin: 'elasticsearch.test.fixtures' - testFixtures.useFixture(fixture.path, 'azure-fixture-repositories-stats') + testFixtures.useFixture(fixture.path, 'azure-fixture-repositories-metering') } integTest { dependsOn repositoryPlugin.bundlePlugin systemProperty 'test.azure.container', azureContainer - nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repositories_stats_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repositories_metering_tests_" + BuildParams.testSeed } testClusters.integTest { testDistribution = 'DEFAULT' plugin repositoryPlugin.bundlePlugin.archiveFile - keystore 'azure.client.repositories_stats.account', azureAccount + keystore 'azure.client.repositories_metering.account', azureAccount if (azureKey != null && azureKey.isEmpty() == false) { - keystore 'azure.client.repositories_stats.key', azureKey + keystore 'azure.client.repositories_metering.key', azureKey } if (azureSasToken != null && azureSasToken.isEmpty() == false) { - keystore 'azure.client.repositories_stats.sas_token', azureSasToken + keystore 'azure.client.repositories_metering.sas_token', azureSasToken } if (useFixture) { @@ -85,8 +85,8 @@ testClusters.integTest { assert ephemeralPort > 0 '127.0.0.1:' + ephemeralPort } - setting 'azure.client.repositories_stats.endpoint_suffix', - { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${-> fixtureAddress('azure-fixture-repositories-stats')}" }, IGNORE_VALUE + setting 'azure.client.repositories_metering.endpoint_suffix', + { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${-> fixtureAddress('azure-fixture-repositories-metering')}" }, IGNORE_VALUE } else { println "Using an external service to test " + project.name diff --git a/x-pack/plugin/repositories-stats/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/stats/azure/AzureRepositoriesStatsIT.java b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metrics/azure/AzureRepositoriesMetricsIT.java similarity index 71% rename from x-pack/plugin/repositories-stats/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/stats/azure/AzureRepositoriesStatsIT.java rename to x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metrics/azure/AzureRepositoriesMetricsIT.java index aed4d77ea100b..37799f195130f 100644 --- a/x-pack/plugin/repositories-stats/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/stats/azure/AzureRepositoriesStatsIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metrics/azure/AzureRepositoriesMetricsIT.java @@ -3,14 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.azure; +package org.elasticsearch.xpack.repositories.metrics.azure; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.repositories.stats.AbstractRepositoriesStatsAPIRestTestCase; +import org.elasticsearch.xpack.repositories.metrics.AbstractRepositoriesMetricsAPIRestTestCase; import java.util.List; -public class AzureRepositoriesStatsIT extends AbstractRepositoriesStatsAPIRestTestCase { +public class AzureRepositoriesMetricsIT extends AbstractRepositoriesMetricsAPIRestTestCase { @Override protected String repositoryType() { @@ -28,12 +28,12 @@ protected Settings repositorySettings() { final String basePath = getProperty("test.azure.base_path"); - return Settings.builder().put("client", "repositories_stats").put("container", container).put("base_path", basePath).build(); + return Settings.builder().put("client", "repositories_metering").put("container", container).put("base_path", basePath).build(); } @Override protected Settings updatedRepositorySettings() { - return Settings.builder().put(repositorySettings()).put("azure.client.repositories_stats.max_retries", 5).build(); + return Settings.builder().put(repositorySettings()).put("azure.client.repositories_metering.max_retries", 5).build(); } @Override diff --git a/x-pack/plugin/repositories-stats/qa/build.gradle b/x-pack/plugin/repositories-metering-api/qa/build.gradle similarity index 100% rename from x-pack/plugin/repositories-stats/qa/build.gradle rename to x-pack/plugin/repositories-metering-api/qa/build.gradle diff --git a/x-pack/plugin/repositories-stats/qa/gcs/build.gradle b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle similarity index 87% rename from x-pack/plugin/repositories-stats/qa/gcs/build.gradle rename to x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle index 0ce6accdd12d1..df34fbc0e8312 100644 --- a/x-pack/plugin/repositories-stats/qa/gcs/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle @@ -33,14 +33,14 @@ final Project fixture = project(':test:fixtures:gcs-fixture') final Project repositoryPlugin = project(':plugins:repository-gcs') dependencies { - testImplementation project(path: xpackModule('repositories-stats'), configuration: 'testArtifacts') + testImplementation project(path: xpackModule('repositories-metering-api'), configuration: 'testArtifacts') testImplementation repositoryPlugin } restResources { restApi { includeCore 'indices', 'bulk', 'snapshot', 'nodes', '_common' - includeXpack 'repositories-stats' + includeXpack 'repositories-metering-api' } } @@ -101,25 +101,25 @@ processTestResources { if (useFixture) { apply plugin: 'elasticsearch.test.fixtures' - testFixtures.useFixture(fixture.path, 'gcs-fixture-repositories-stats') + testFixtures.useFixture(fixture.path, 'gcs-fixture-repositories-metering') } integTest { dependsOn repositoryPlugin.bundlePlugin systemProperty 'test.gcs.bucket', gcsBucket - nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repositories_stats" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repositories_metering" + BuildParams.testSeed } testClusters.integTest { testDistribution = 'DEFAULT' plugin repositoryPlugin.bundlePlugin.archiveFile - keystore 'gcs.client.repositories_stats.credentials_file', serviceAccountFile, IGNORE_VALUE + keystore 'gcs.client.repositories_metering.credentials_file', serviceAccountFile, IGNORE_VALUE if (useFixture) { tasks.integTest.dependsOn createServiceAccountFile /* Use a closure on the string to delay evaluation until tests are executed */ - setting 'gcs.client.repositories_stats.endpoint', { "${-> fixtureAddress('gcs-fixture-repositories-stats')}" }, IGNORE_VALUE - setting 'gcs.client.repositories_stats.token_uri', { "${-> fixtureAddress('gcs-fixture-repositories-stats')}/o/oauth2/token" }, IGNORE_VALUE + setting 'gcs.client.repositories_metering.endpoint', { "${-> fixtureAddress('gcs-fixture-repositories-metering')}" }, IGNORE_VALUE + setting 'gcs.client.repositories_metering.token_uri', { "${-> fixtureAddress('gcs-fixture-repositories-metering')}/o/oauth2/token" }, IGNORE_VALUE } else { println "Using an external service to test " + project.name } diff --git a/x-pack/plugin/repositories-stats/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/stats/gcs/GCSRepositoriesStatsIT.java b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metrics/gcs/GCSRepositoriesMetricsIT.java similarity index 71% rename from x-pack/plugin/repositories-stats/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/stats/gcs/GCSRepositoriesStatsIT.java rename to x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metrics/gcs/GCSRepositoriesMetricsIT.java index f96238804a47d..51160607ed914 100644 --- a/x-pack/plugin/repositories-stats/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/stats/gcs/GCSRepositoriesStatsIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metrics/gcs/GCSRepositoriesMetricsIT.java @@ -3,14 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.gcs; +package org.elasticsearch.xpack.repositories.metrics.gcs; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.repositories.stats.AbstractRepositoriesStatsAPIRestTestCase; +import org.elasticsearch.xpack.repositories.metrics.AbstractRepositoriesMetricsAPIRestTestCase; import java.util.List; -public class GCSRepositoriesStatsIT extends AbstractRepositoriesStatsAPIRestTestCase { +public class GCSRepositoriesMetricsIT extends AbstractRepositoriesMetricsAPIRestTestCase { @Override protected String repositoryType() { @@ -27,12 +27,12 @@ protected Settings repositorySettings() { final String bucket = getProperty("test.gcs.bucket"); final String basePath = getProperty("test.gcs.base_path"); - return Settings.builder().put("client", "repositories_stats").put("bucket", bucket).put("base_path", basePath).build(); + return Settings.builder().put("client", "repositories_metering").put("bucket", bucket).put("base_path", basePath).build(); } @Override protected Settings updatedRepositorySettings() { - return Settings.builder().put(repositorySettings()).put("gcs.client.repositories_stats.application_name", "updated").build(); + return Settings.builder().put(repositorySettings()).put("gcs.client.repositories_metering.application_name", "updated").build(); } @Override diff --git a/x-pack/plugin/repositories-stats/qa/s3/build.gradle b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle similarity index 90% rename from x-pack/plugin/repositories-stats/qa/s3/build.gradle rename to x-pack/plugin/repositories-metering-api/qa/s3/build.gradle index 1c0ed18b01db1..63e2d46bb0321 100644 --- a/x-pack/plugin/repositories-stats/qa/s3/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle @@ -9,14 +9,14 @@ final Project fixture = project(':test:fixtures:s3-fixture') final Project repositoryPlugin = project(':plugins:repository-s3') dependencies { - testImplementation project(path: xpackModule('repositories-stats'), configuration: 'testArtifacts') + testImplementation project(path: xpackModule('repositories-metering-api'), configuration: 'testArtifacts') testImplementation repositoryPlugin } restResources { restApi { includeCore 'indices', 'bulk', 'snapshot', 'nodes', '_common' - includeXpack 'repositories-stats-api' + includeXpack 'repositories-metering-api' } } @@ -39,7 +39,7 @@ if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) { if (useFixture) { apply plugin: 'elasticsearch.test.fixtures' - testFixtures.useFixture(fixture.path, 's3-fixture-repositories-stats') + testFixtures.useFixture(fixture.path, 's3-fixture-repositories-metering') } integTest { @@ -63,7 +63,7 @@ testClusters.integTest { '127.0.0.1:' + ephemeralPort } setting 's3.client.repositories_metering.protocol', 'http' - setting 's3.client.repositories_metering.endpoint', { "${-> fixtureAddress('s3-fixture-repositories-stats')}" }, IGNORE_VALUE + setting 's3.client.repositories_metering.endpoint', { "${-> fixtureAddress('s3-fixture-repositories-metering')}" }, IGNORE_VALUE } else { println "Using an external service to test " + project.name diff --git a/x-pack/plugin/repositories-stats/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/stats/s3/S3RepositoriesStatsIT.java b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metrics/s3/S3RepositoriesMetricsIT.java similarity index 84% rename from x-pack/plugin/repositories-stats/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/stats/s3/S3RepositoriesStatsIT.java rename to x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metrics/s3/S3RepositoriesMetricsIT.java index d88bcb896daf1..6e548c4df25e1 100644 --- a/x-pack/plugin/repositories-stats/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/stats/s3/S3RepositoriesStatsIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metrics/s3/S3RepositoriesMetricsIT.java @@ -3,14 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.s3; +package org.elasticsearch.xpack.repositories.metrics.s3; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.repositories.stats.AbstractRepositoriesStatsAPIRestTestCase; +import org.elasticsearch.xpack.repositories.metrics.AbstractRepositoriesMetricsAPIRestTestCase; import java.util.List; -public class S3RepositoriesStatsIT extends AbstractRepositoriesStatsAPIRestTestCase { +public class S3RepositoriesMetricsIT extends AbstractRepositoriesMetricsAPIRestTestCase { @Override protected String repositoryType() { diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/RepositoriesStatsPlugin.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/RepositoriesMeteringPlugin.java similarity index 57% rename from x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/RepositoriesStatsPlugin.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/RepositoriesMeteringPlugin.java index 239c01e5e2173..a687885772fab 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/RepositoriesStatsPlugin.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/RepositoriesMeteringPlugin.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats; +package org.elasticsearch.xpack.repositories.metrics; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -18,23 +18,23 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.xpack.repositories.stats.action.ClearRepositoriesStatsArchiveAction; -import org.elasticsearch.xpack.repositories.stats.action.RepositoriesStatsAction; -import org.elasticsearch.xpack.repositories.stats.action.TransportClearRepositoriesStatsArchiveAction; -import org.elasticsearch.xpack.repositories.stats.action.TransportRepositoriesStatsAction; -import org.elasticsearch.xpack.repositories.stats.rest.RestClearRepositoriesStatsArchiveAction; -import org.elasticsearch.xpack.repositories.stats.rest.RestGetRepositoriesStatsAction; +import org.elasticsearch.xpack.repositories.metrics.action.ClearRepositoriesMetricsArchiveAction; +import org.elasticsearch.xpack.repositories.metrics.action.RepositoriesMetricsAction; +import org.elasticsearch.xpack.repositories.metrics.action.TransportClearRepositoriesStatsArchiveAction; +import org.elasticsearch.xpack.repositories.metrics.action.TransportRepositoriesStatsAction; +import org.elasticsearch.xpack.repositories.metrics.rest.RestClearRepositoriesMetricsArchiveAction; +import org.elasticsearch.xpack.repositories.metrics.rest.RestGetRepositoriesMetricsAction; import java.util.List; import java.util.function.Supplier; -public final class RepositoriesStatsPlugin extends Plugin implements ActionPlugin { +public final class RepositoriesMeteringPlugin extends Plugin implements ActionPlugin { @Override public List> getActions() { return List.of( - new ActionHandler<>(RepositoriesStatsAction.INSTANCE, TransportRepositoriesStatsAction.class), - new ActionHandler<>(ClearRepositoriesStatsArchiveAction.INSTANCE, TransportClearRepositoriesStatsArchiveAction.class) + new ActionHandler<>(RepositoriesMetricsAction.INSTANCE, TransportRepositoriesStatsAction.class), + new ActionHandler<>(ClearRepositoriesMetricsArchiveAction.INSTANCE, TransportClearRepositoriesStatsArchiveAction.class) ); } @@ -48,7 +48,6 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - - return List.of(new RestGetRepositoriesStatsAction(), new RestClearRepositoriesStatsArchiveAction()); + return List.of(new RestGetRepositoriesMetricsAction(), new RestClearRepositoriesMetricsArchiveAction()); } } diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveAction.java new file mode 100644 index 0000000000000..f1bef39073a0f --- /dev/null +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveAction.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.metrics.action; + +import org.elasticsearch.action.ActionType; + +public final class ClearRepositoriesMetricsArchiveAction extends ActionType { + public static final ClearRepositoriesMetricsArchiveAction INSTANCE = new ClearRepositoriesMetricsArchiveAction(); + + static final String NAME = "cluster:monitor/xpack/repositories_metrics/clear_repositories_metrics_archive"; + + ClearRepositoriesMetricsArchiveAction() { + super(NAME, RepositoriesMetricsResponse::new); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveRequest.java similarity index 70% rename from x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveRequest.java index ab9e414fed2f9..43ec61700160d 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveRequest.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveRequest.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.action; +package org.elasticsearch.xpack.repositories.metrics.action; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; @@ -12,15 +12,15 @@ import java.io.IOException; -public final class ClearRepositoriesStatsArchiveRequest extends BaseNodesRequest { +public final class ClearRepositoriesMetricsArchiveRequest extends BaseNodesRequest { private final long maxVersionToClear; - public ClearRepositoriesStatsArchiveRequest(StreamInput in) throws IOException { + public ClearRepositoriesMetricsArchiveRequest(StreamInput in) throws IOException { super(in); this.maxVersionToClear = in.readLong(); } - public ClearRepositoriesStatsArchiveRequest(long maxVersionToClear, String... nodesIds) { + public ClearRepositoriesMetricsArchiveRequest(long maxVersionToClear, String... nodesIds) { super(nodesIds); this.maxVersionToClear = maxVersionToClear; } diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsAction.java new file mode 100644 index 0000000000000..f8850c361e88d --- /dev/null +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsAction.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.metrics.action; + +import org.elasticsearch.action.ActionType; + +public final class RepositoriesMetricsAction extends ActionType { + public static final RepositoriesMetricsAction INSTANCE = new RepositoriesMetricsAction(); + + static final String NAME = "cluster:monitor/xpack/repositories_metrics/get_metrics"; + + RepositoriesMetricsAction() { + super(NAME, RepositoriesMetricsResponse::new); + } +} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsRequest.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsRequest.java similarity index 60% rename from x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsRequest.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsRequest.java index 562281ffa6324..f887a8c27db0d 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsRequest.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsRequest.java @@ -4,19 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.action; +package org.elasticsearch.xpack.repositories.metrics.action; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import java.io.IOException; -public final class RepositoriesStatsRequest extends BaseNodesRequest { - public RepositoriesStatsRequest(StreamInput in) throws IOException { +public final class RepositoriesMetricsRequest extends BaseNodesRequest { + public RepositoriesMetricsRequest(StreamInput in) throws IOException { super(in); } - public RepositoriesStatsRequest(String... nodesIds) { + public RepositoriesMetricsRequest(String... nodesIds) { super(nodesIds); } } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponse.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponse.java similarity index 65% rename from x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponse.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponse.java index ecdcedf534377..650ae0a158115 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponse.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponse.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.action; +package org.elasticsearch.xpack.repositories.metrics.action; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.nodes.BaseNodesResponse; @@ -17,34 +17,34 @@ import java.io.IOException; import java.util.List; -public final class RepositoriesStatsResponse extends BaseNodesResponse implements ToXContentFragment { +public final class RepositoriesMetricsResponse extends BaseNodesResponse implements ToXContentFragment { - public RepositoriesStatsResponse(StreamInput in) throws IOException { + public RepositoriesMetricsResponse(StreamInput in) throws IOException { super(in); } - public RepositoriesStatsResponse( + public RepositoriesMetricsResponse( ClusterName clusterName, - List nodes, + List nodes, List failures ) { super(clusterName, nodes, failures); } @Override - protected List readNodesFrom(StreamInput in) throws IOException { - return in.readList(RepositoriesNodeStatsResponse::new); + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(RepositoriesNodeMetricsResponse::new); } @Override - protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { out.writeList(nodes); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("nodes"); - for (RepositoriesNodeStatsResponse nodeStats : getNodes()) { + for (RepositoriesNodeMetricsResponse nodeStats : getNodes()) { nodeStats.toXContent(builder, params); } builder.endObject(); diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsResponse.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesNodeMetricsResponse.java similarity index 80% rename from x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsResponse.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesNodeMetricsResponse.java index fcda075d572f3..6f08becc7a656 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesNodeStatsResponse.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesNodeMetricsResponse.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.action; +package org.elasticsearch.xpack.repositories.metrics.action; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -18,16 +18,16 @@ import java.io.IOException; import java.util.List; -public final class RepositoriesNodeStatsResponse extends BaseNodeResponse implements ToXContentFragment { +public final class RepositoriesNodeMetricsResponse extends BaseNodeResponse implements ToXContentFragment { final List repositoryStatsSnapshots; - public RepositoriesNodeStatsResponse(DiscoveryNode node, List repositoryStatsSnapshots) { + public RepositoriesNodeMetricsResponse(DiscoveryNode node, List repositoryStatsSnapshots) { super(node); this.repositoryStatsSnapshots = repositoryStatsSnapshots; } - public RepositoriesNodeStatsResponse(StreamInput in) throws IOException { + public RepositoriesNodeMetricsResponse(StreamInput in) throws IOException { super(in); this.repositoryStatsSnapshots = in.readList(RepositoryStatsSnapshot::new); } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportClearRepositoriesStatsArchiveAction.java similarity index 73% rename from x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportClearRepositoriesStatsArchiveAction.java index 4ec883253ba1e..6084c80d973f5 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportClearRepositoriesStatsArchiveAction.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.action; +package org.elasticsearch.xpack.repositories.metrics.action; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; @@ -24,10 +24,10 @@ import java.util.List; public final class TransportClearRepositoriesStatsArchiveAction extends TransportNodesAction< - ClearRepositoriesStatsArchiveRequest, - RepositoriesStatsResponse, + ClearRepositoriesMetricsArchiveRequest, + RepositoriesMetricsResponse, TransportClearRepositoriesStatsArchiveAction.ClearRepositoriesStatsArchiveNodeRequest, - RepositoriesNodeStatsResponse> { + RepositoriesNodeMetricsResponse> { private final RepositoriesService repositoriesService; @@ -40,42 +40,42 @@ public TransportClearRepositoriesStatsArchiveAction( RepositoriesService repositoriesService ) { super( - ClearRepositoriesStatsArchiveAction.NAME, + ClearRepositoriesMetricsArchiveAction.NAME, threadPool, clusterService, transportService, actionFilters, - ClearRepositoriesStatsArchiveRequest::new, + ClearRepositoriesMetricsArchiveRequest::new, ClearRepositoriesStatsArchiveNodeRequest::new, ThreadPool.Names.SAME, - RepositoriesNodeStatsResponse.class + RepositoriesNodeMetricsResponse.class ); this.repositoriesService = repositoriesService; } @Override - protected RepositoriesStatsResponse newResponse( - ClearRepositoriesStatsArchiveRequest request, - List nodesResponses, + protected RepositoriesMetricsResponse newResponse( + ClearRepositoriesMetricsArchiveRequest request, + List nodesResponses, List failures ) { - return new RepositoriesStatsResponse(clusterService.getClusterName(), nodesResponses, failures); + return new RepositoriesMetricsResponse(clusterService.getClusterName(), nodesResponses, failures); } @Override - protected ClearRepositoriesStatsArchiveNodeRequest newNodeRequest(ClearRepositoriesStatsArchiveRequest request) { + protected ClearRepositoriesStatsArchiveNodeRequest newNodeRequest(ClearRepositoriesMetricsArchiveRequest request) { return new ClearRepositoriesStatsArchiveNodeRequest(request.getMaxVersionToClear()); } @Override - protected RepositoriesNodeStatsResponse newNodeResponse(StreamInput in) throws IOException { - return new RepositoriesNodeStatsResponse(in); + protected RepositoriesNodeMetricsResponse newNodeResponse(StreamInput in) throws IOException { + return new RepositoriesNodeMetricsResponse(in); } @Override - protected RepositoriesNodeStatsResponse nodeOperation(ClearRepositoriesStatsArchiveNodeRequest request, Task task) { + protected RepositoriesNodeMetricsResponse nodeOperation(ClearRepositoriesStatsArchiveNodeRequest request, Task task) { List clearedStats = repositoriesService.clearRepositoriesStatsArchive(request.maxVersionToClear); - return new RepositoriesNodeStatsResponse(clusterService.localNode(), clearedStats); + return new RepositoriesNodeMetricsResponse(clusterService.localNode(), clearedStats); } static final class ClearRepositoriesStatsArchiveNodeRequest extends TransportRequest { diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportRepositoriesStatsAction.java similarity index 67% rename from x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportRepositoriesStatsAction.java index 35346cdc724f8..7e0092dcb626b 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/TransportRepositoriesStatsAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportRepositoriesStatsAction.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.action; +package org.elasticsearch.xpack.repositories.metrics.action; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; @@ -22,10 +22,10 @@ import java.util.List; public final class TransportRepositoriesStatsAction extends TransportNodesAction< - RepositoriesStatsRequest, - RepositoriesStatsResponse, + RepositoriesMetricsRequest, + RepositoriesMetricsResponse, TransportRepositoriesStatsAction.RepositoriesNodeStatsRequest, - RepositoriesNodeStatsResponse> { + RepositoriesNodeMetricsResponse> { private final RepositoriesService repositoriesService; @@ -38,41 +38,41 @@ public TransportRepositoriesStatsAction( RepositoriesService repositoriesService ) { super( - RepositoriesStatsAction.NAME, + RepositoriesMetricsAction.NAME, threadPool, clusterService, transportService, actionFilters, - RepositoriesStatsRequest::new, + RepositoriesMetricsRequest::new, RepositoriesNodeStatsRequest::new, ThreadPool.Names.SAME, - RepositoriesNodeStatsResponse.class + RepositoriesNodeMetricsResponse.class ); this.repositoriesService = repositoriesService; } @Override - protected RepositoriesStatsResponse newResponse( - RepositoriesStatsRequest request, - List repositoriesNodeStatsResponses, + protected RepositoriesMetricsResponse newResponse( + RepositoriesMetricsRequest request, + List repositoriesNodeStatsResponses, List failures ) { - return new RepositoriesStatsResponse(clusterService.getClusterName(), repositoriesNodeStatsResponses, failures); + return new RepositoriesMetricsResponse(clusterService.getClusterName(), repositoriesNodeStatsResponses, failures); } @Override - protected RepositoriesNodeStatsRequest newNodeRequest(RepositoriesStatsRequest request) { + protected RepositoriesNodeStatsRequest newNodeRequest(RepositoriesMetricsRequest request) { return new RepositoriesNodeStatsRequest(); } @Override - protected RepositoriesNodeStatsResponse newNodeResponse(StreamInput in) throws IOException { - return new RepositoriesNodeStatsResponse(in); + protected RepositoriesNodeMetricsResponse newNodeResponse(StreamInput in) throws IOException { + return new RepositoriesNodeMetricsResponse(in); } @Override - protected RepositoriesNodeStatsResponse nodeOperation(RepositoriesNodeStatsRequest request, Task task) { - return new RepositoriesNodeStatsResponse(clusterService.localNode(), repositoriesService.repositoriesStats()); + protected RepositoriesNodeMetricsResponse nodeOperation(RepositoriesNodeStatsRequest request, Task task) { + return new RepositoriesNodeMetricsResponse(clusterService.localNode(), repositoriesService.repositoriesStats()); } static final class RepositoriesNodeStatsRequest extends TransportRequest { diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java similarity index 62% rename from x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java index 658ca16de9999..9a259ce070f76 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java @@ -4,36 +4,36 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.rest; +package org.elasticsearch.xpack.repositories.metrics.rest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; -import org.elasticsearch.xpack.repositories.stats.action.ClearRepositoriesStatsArchiveAction; -import org.elasticsearch.xpack.repositories.stats.action.ClearRepositoriesStatsArchiveRequest; +import org.elasticsearch.xpack.repositories.metrics.action.ClearRepositoriesMetricsArchiveAction; +import org.elasticsearch.xpack.repositories.metrics.action.ClearRepositoriesMetricsArchiveRequest; import java.util.List; -public class RestClearRepositoriesStatsArchiveAction extends BaseRestHandler { +public class RestClearRepositoriesMetricsArchiveAction extends BaseRestHandler { @Override public String getName() { - return "clear_repositories_stats_action"; + return "clear_repositories_metrics_archive_action"; } @Override public List routes() { - return List.of(new Route(RestRequest.Method.DELETE, "/_nodes/{nodeId}/_repositories_stats/{maxVersionToClear}")); + return List.of(new Route(RestRequest.Method.DELETE, "/_nodes/{nodeId}/_repositories_metrics/{maxVersionToClear}")); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); long maxVersionToClear = request.paramAsLong("maxVersionToClear", -1); - ClearRepositoriesStatsArchiveRequest clearArchivesRequest = new ClearRepositoriesStatsArchiveRequest(maxVersionToClear, nodesIds); + ClearRepositoriesMetricsArchiveRequest clearArchivesRequest = new ClearRepositoriesMetricsArchiveRequest(maxVersionToClear, nodesIds); return channel -> client.execute( - ClearRepositoriesStatsArchiveAction.INSTANCE, + ClearRepositoriesMetricsArchiveAction.INSTANCE, clearArchivesRequest, new RestActions.NodesResponseRestListener<>(channel) ); diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestGetRepositoriesStatsAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestGetRepositoriesMetricsAction.java similarity index 62% rename from x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestGetRepositoriesStatsAction.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestGetRepositoriesMetricsAction.java index 0c28a2ef9d608..67684b4ba49a6 100644 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/rest/RestGetRepositoriesStatsAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestGetRepositoriesMetricsAction.java @@ -4,37 +4,37 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.rest; +package org.elasticsearch.xpack.repositories.metrics.rest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; -import org.elasticsearch.xpack.repositories.stats.action.RepositoriesStatsRequest; -import org.elasticsearch.xpack.repositories.stats.action.RepositoriesStatsAction; +import org.elasticsearch.xpack.repositories.metrics.action.RepositoriesMetricsRequest; +import org.elasticsearch.xpack.repositories.metrics.action.RepositoriesMetricsAction; import java.util.List; -public final class RestGetRepositoriesStatsAction extends BaseRestHandler { +public final class RestGetRepositoriesMetricsAction extends BaseRestHandler { @Override public String getName() { - return "get_repositories_stats_action"; + return "get_repositories_metrics_action"; } @Override public List routes() { - return List.of(new Route(RestRequest.Method.GET, "/_nodes/{nodeId}/_repositories_stats")); + return List.of(new Route(RestRequest.Method.GET, "/_nodes/{nodeId}/_repositories_metrics")); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); - RepositoriesStatsRequest repositoriesStatsRequest = new RepositoriesStatsRequest(nodesIds); + RepositoriesMetricsRequest repositoriesMetricsRequest = new RepositoriesMetricsRequest(nodesIds); return channel -> client.execute( - RepositoriesStatsAction.INSTANCE, - repositoriesStatsRequest, + RepositoriesMetricsAction.INSTANCE, + repositoriesMetricsRequest, new RestActions.NodesResponseRestListener<>(channel) ); } diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/AbstractRepositoriesMetricsAPIRestTestCase.java similarity index 98% rename from x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java rename to x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/AbstractRepositoriesMetricsAPIRestTestCase.java index db17bfc719b44..7c90acef78a0c 100644 --- a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/AbstractRepositoriesStatsAPIRestTestCase.java +++ b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/AbstractRepositoriesMetricsAPIRestTestCase.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats; +package org.elasticsearch.xpack.repositories.metrics; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpPost; @@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; -public abstract class AbstractRepositoriesStatsAPIRestTestCase extends ESRestTestCase { +public abstract class AbstractRepositoriesMetricsAPIRestTestCase extends ESRestTestCase { protected abstract String repositoryType(); protected abstract String repositoryLocation(); @@ -299,7 +299,7 @@ private void assertAllRequestCountsAreZero(RepositoryStatsSnapshot statsSnapshot } private List getRepositoriesStats() throws IOException { - Map response = getAsMap("/_nodes/_all/_repositories_stats"); + Map response = getAsMap("/_nodes/_all/_repositories_metrics"); return parseRepositoriesStatsResponse(response); } @@ -348,7 +348,7 @@ private Set getNodeIds() throws IOException { } private List clearRepositoriesStats(long maxVersionToClear) throws IOException { - final Request request = new Request(HttpDelete.METHOD_NAME, "/_nodes/_all/_repositories_stats/" + maxVersionToClear); + final Request request = new Request(HttpDelete.METHOD_NAME, "/_nodes/_all/_repositories_metrics/" + maxVersionToClear); final Response response = client().performRequest(request); assertThat( "Failed to clear repositories stats: " + response, diff --git a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponseTests.java similarity index 74% rename from x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java rename to x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponseTests.java index 8637c9f64c9f5..19b5dea0e31a5 100644 --- a/x-pack/plugin/repositories-stats/src/test/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsResponseTests.java +++ b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponseTests.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.stats.action; +package org.elasticsearch.xpack.repositories.metrics.action; import org.elasticsearch.Version; import org.elasticsearch.action.FailedNodeException; @@ -22,25 +22,25 @@ import static org.hamcrest.Matchers.equalTo; -public class RepositoriesStatsResponseTests extends ESTestCase { +public class RepositoriesMetricsResponseTests extends ESTestCase { public void testSerializationRoundtrip() throws Exception { - final RepositoriesStatsResponse repositoriesStatsResponse = createResponse(); - final RepositoriesStatsResponse deserializedResponse = copyWriteable( - repositoriesStatsResponse, + final RepositoriesMetricsResponse repositoriesMetricsResponse = createResponse(); + final RepositoriesMetricsResponse deserializedResponse = copyWriteable( + repositoriesMetricsResponse, writableRegistry(), - RepositoriesStatsResponse::new, + RepositoriesMetricsResponse::new, Version.CURRENT ); - assertResponsesAreEqual(repositoriesStatsResponse, deserializedResponse); + assertResponsesAreEqual(repositoriesMetricsResponse, deserializedResponse); } - private void assertResponsesAreEqual(RepositoriesStatsResponse response, RepositoriesStatsResponse otherResponse) { - List nodeResponses = response.getNodes(); - List otherNodeResponses = otherResponse.getNodes(); + private void assertResponsesAreEqual(RepositoriesMetricsResponse response, RepositoriesMetricsResponse otherResponse) { + List nodeResponses = response.getNodes(); + List otherNodeResponses = otherResponse.getNodes(); assertThat(nodeResponses.size(), equalTo(otherNodeResponses.size())); for (int i = 0; i < nodeResponses.size(); i++) { - RepositoriesNodeStatsResponse nodeResponse = nodeResponses.get(i); - RepositoriesNodeStatsResponse otherNodeResponse = otherNodeResponses.get(i); + RepositoriesNodeMetricsResponse nodeResponse = nodeResponses.get(i); + RepositoriesNodeMetricsResponse otherNodeResponse = otherNodeResponses.get(i); assertThat(nodeResponse.repositoryStatsSnapshots, equalTo(otherNodeResponse.repositoryStatsSnapshots)); } @@ -55,10 +55,10 @@ private void assertResponsesAreEqual(RepositoriesStatsResponse response, Reposit } } - private RepositoriesStatsResponse createResponse() { + private RepositoriesMetricsResponse createResponse() { ClusterName clusterName = new ClusterName("test"); int nodes = randomIntBetween(1, 10); - List nodeResponses = new ArrayList<>(nodes); + List nodeResponses = new ArrayList<>(nodes); for (int i = 0; i < nodes; i++) { DiscoveryNode node = new DiscoveryNode("nodeId" + i, buildNewFakeTransportAddress(), Version.CURRENT); int numberOfRepos = randomInt(10); @@ -81,7 +81,7 @@ private RepositoriesStatsResponse createResponse() { nodeRepoStats.add(statsSnapshot); } - nodeResponses.add(new RepositoriesNodeStatsResponse(node, nodeRepoStats)); + nodeResponses.add(new RepositoriesNodeMetricsResponse(node, nodeRepoStats)); } int numberOfFailures = randomInt(20); @@ -95,6 +95,6 @@ private RepositoriesStatsResponse createResponse() { failures.add(failedNodeException); } - return new RepositoriesStatsResponse(clusterName, nodeResponses, failures); + return new RepositoriesMetricsResponse(clusterName, nodeResponses, failures); } } diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java deleted file mode 100644 index 5ebf2f56bf12f..0000000000000 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/ClearRepositoriesStatsArchiveAction.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.repositories.stats.action; - -import org.elasticsearch.action.ActionType; - -public final class ClearRepositoriesStatsArchiveAction extends ActionType { - public static final ClearRepositoriesStatsArchiveAction INSTANCE = new ClearRepositoriesStatsArchiveAction(); - - static final String NAME = "cluster:monitor/xpack/repositories_stats/clear_repositories_stats_archive"; - - ClearRepositoriesStatsArchiveAction() { - super(NAME, RepositoriesStatsResponse::new); - } -} diff --git a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsAction.java b/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsAction.java deleted file mode 100644 index a554a26c252fa..0000000000000 --- a/x-pack/plugin/repositories-stats/src/main/java/org/elasticsearch/xpack/repositories/stats/action/RepositoriesStatsAction.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.repositories.stats.action; - -import org.elasticsearch.action.ActionType; - -public final class RepositoriesStatsAction extends ActionType { - public static final RepositoriesStatsAction INSTANCE = new RepositoriesStatsAction(); - - static final String NAME = "cluster:monitor/xpack/repositories_stats/get_stats"; - - RepositoriesStatsAction() { - super(NAME, RepositoriesStatsResponse::new); - } -} From 7e4ab0ca9f4eacc96cfc8585448f96e801e5f519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 11 Aug 2020 14:29:54 +0200 Subject: [PATCH 09/19] Fix styleCheck --- .../rest/RestClearRepositoriesMetricsArchiveAction.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java index 9a259ce070f76..ab38859ae4294 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java @@ -31,7 +31,10 @@ public List routes() { protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); long maxVersionToClear = request.paramAsLong("maxVersionToClear", -1); - ClearRepositoriesMetricsArchiveRequest clearArchivesRequest = new ClearRepositoriesMetricsArchiveRequest(maxVersionToClear, nodesIds); + ClearRepositoriesMetricsArchiveRequest clearArchivesRequest = new ClearRepositoriesMetricsArchiveRequest( + maxVersionToClear, + nodesIds + ); return channel -> client.execute( ClearRepositoriesMetricsArchiveAction.INSTANCE, clearArchivesRequest, From d7020cb2e3aef959d1976b0954569acc3b12f561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 18 Aug 2020 13:54:47 +0200 Subject: [PATCH 10/19] Rename operation names on repositories metering --- ...ear-repositories-metering-archive.asciidoc | 39 +++++++++++++ ...lear-repositories-metrics-archive.asciidoc | 37 ------------ ...doc => get-repositories-metering.asciidoc} | 57 +++++++++++-------- .../repositories-metering.asciidoc | 10 ++-- .../azure/AzureBlobStoreRepositoryTests.java | 10 ++-- .../repositories/azure/AzureBlobStore.java | 10 ++-- ...eCloudStorageBlobStoreRepositoryTests.java | 10 ++-- .../GoogleCloudStorageOperationsStats.java | 8 +-- .../s3/S3BlobStoreRepositoryTests.java | 8 +-- .../repositories/s3/S3BlobStore.java | 8 +-- .../repositories/RepositoriesService.java | 4 +- .../repositories/RepositoryStatsSnapshot.java | 6 +- .../blobstore/MeteredBlobStoreRepository.java | 4 +- .../repositories-metering-api/build.gradle | 2 +- .../azure/AzureRepositoriesMeteringIT.java} | 6 +- .../gcs/GCSRepositoriesMeteringIT.java} | 6 +- .../s3/S3RepositoriesMeteringIT.java} | 6 +- .../RepositoriesMeteringPlugin.java | 20 +++---- ...learRepositoriesMeteringArchiveAction.java | 19 +++++++ ...arRepositoriesMeteringArchiveRequest.java} | 8 +-- .../action/RepositoriesMeteringAction.java | 19 +++++++ .../action/RepositoriesMeteringRequest.java} | 8 +-- .../action/RepositoriesMeteringResponse.java} | 18 +++--- .../RepositoriesNodeMeteringResponse.java} | 8 +-- ...rtClearRepositoriesStatsArchiveAction.java | 32 +++++------ .../TransportRepositoriesStatsAction.java | 32 +++++------ ...earRepositoriesMeteringArchiveAction.java} | 14 ++--- .../RestGetRepositoriesMeteringAction.java} | 18 +++--- ...ClearRepositoriesMetricsArchiveAction.java | 19 ------- .../action/RepositoriesMetricsAction.java | 19 ------- ...tRepositoriesMeteringAPIRestTestCase.java} | 13 +++-- .../RepositoriesMeteringResponseTests.java} | 37 ++++++------ 32 files changed, 266 insertions(+), 249 deletions(-) create mode 100644 docs/reference/repositories-metering/apis/clear-repositories-metering-archive.asciidoc delete mode 100644 docs/reference/repositories-metering/apis/clear-repositories-metrics-archive.asciidoc rename docs/reference/repositories-metering/apis/{get-repositories-metrics.asciidoc => get-repositories-metering.asciidoc} (72%) rename x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/{metrics/azure/AzureRepositoriesMetricsIT.java => metering/azure/AzureRepositoriesMeteringIT.java} (83%) rename x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/{metrics/gcs/GCSRepositoriesMetricsIT.java => metering/gcs/GCSRepositoriesMeteringIT.java} (83%) rename x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/{metrics/s3/S3RepositoriesMetricsIT.java => metering/s3/S3RepositoriesMeteringIT.java} (84%) rename x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/{metrics => metering}/RepositoriesMeteringPlugin.java (60%) create mode 100644 x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveAction.java rename x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/{metrics/action/ClearRepositoriesMetricsArchiveRequest.java => metering/action/ClearRepositoriesMeteringArchiveRequest.java} (70%) create mode 100644 x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringAction.java rename x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/{metrics/action/RepositoriesMetricsRequest.java => metering/action/RepositoriesMeteringRequest.java} (59%) rename x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/{metrics/action/RepositoriesMetricsResponse.java => metering/action/RepositoriesMeteringResponse.java} (65%) rename x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/{metrics/action/RepositoriesNodeMetricsResponse.java => metering/action/RepositoriesNodeMeteringResponse.java} (80%) rename x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/{metrics => metering}/action/TransportClearRepositoriesStatsArchiveAction.java (72%) rename x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/{metrics => metering}/action/TransportRepositoriesStatsAction.java (67%) rename x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/{metrics/rest/RestClearRepositoriesMetricsArchiveAction.java => metering/rest/RestClearRepositoriesMeteringArchiveAction.java} (67%) rename x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/{metrics/rest/RestGetRepositoriesMetricsAction.java => metering/rest/RestGetRepositoriesMeteringAction.java} (62%) delete mode 100644 x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveAction.java delete mode 100644 x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsAction.java rename x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/{metrics/AbstractRepositoriesMetricsAPIRestTestCase.java => metering/AbstractRepositoriesMeteringAPIRestTestCase.java} (97%) rename x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/{metrics/action/RepositoriesMetricsResponseTests.java => metering/action/RepositoriesMeteringResponseTests.java} (71%) diff --git a/docs/reference/repositories-metering/apis/clear-repositories-metering-archive.asciidoc b/docs/reference/repositories-metering/apis/clear-repositories-metering-archive.asciidoc new file mode 100644 index 0000000000000..c12c5c15ae452 --- /dev/null +++ b/docs/reference/repositories-metering/apis/clear-repositories-metering-archive.asciidoc @@ -0,0 +1,39 @@ +[role="xpack"] +[testenv="basic"] +[[clear-repositories-metering-archive-api]] +=== Clear repositories metering archive API +++++ +Clear repositories metering archive API +++++ + +Removes the archived repositories metrics in the cluster up to +a particular version. + +[[clear-repositories-metering-archive-api-request]] +==== {api-request-title} + +`DELETE /_nodes//_repositories_metering/` + +[[clear-repositories-metering-archive-api-desc]] +==== {api-description-title} + +You can clear the archived repositories utilization metrics using this API +up to a version. + +All the nodes selective options are explained <>. + +[[clear-repositories-metering-archive-api-path-params]] +==== {api-path-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] + +``:: + (long) Specify the max version to be cleared from the archive. + +[[clear-repositories-metering-archive-api-query-params]] +==== {api-query-parms-title} + +[role="child_attributes"] +[[clear-repositories-metering-archive-api-response-body]] +==== {api-response-body-title} +It returns all the deleted repositories metrics. diff --git a/docs/reference/repositories-metering/apis/clear-repositories-metrics-archive.asciidoc b/docs/reference/repositories-metering/apis/clear-repositories-metrics-archive.asciidoc deleted file mode 100644 index c5875a4591d97..0000000000000 --- a/docs/reference/repositories-metering/apis/clear-repositories-metrics-archive.asciidoc +++ /dev/null @@ -1,37 +0,0 @@ -[[clear-repositories-metrics-archive-api]] -=== Clear repositories metrics archive API -++++ -Clear repositories metrics archive API -++++ - -Removes the archived repositories metrics in the cluster up to -a particular version. - -[[clear-repositories-metrics-archive-api-request]] -==== {api-request-title} - -`DELETE /_nodes//_repositories_metrics/` - -[[clear-repositories-metrics-archive-api-desc]] -==== {api-description-title} - -You can clear the archived repositories utilization statistics using this API -up to a version. - -All the nodes selective options are explained <>. - -[[clear-repositories-metrics-archive-api-path-params]] -==== {api-path-parms-title} - -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] - -``:: - (long) Specify the max version to be cleared from the archive. - -[[clear-repositories-metrics-archive-api-query-params]] -==== {api-query-parms-title} - -[role="child_attributes"] -[[clear-repositories-metrics-archive-api-response-body]] -==== {api-response-body-title} -It returns all the deleted repositories metrics. diff --git a/docs/reference/repositories-metering/apis/get-repositories-metrics.asciidoc b/docs/reference/repositories-metering/apis/get-repositories-metering.asciidoc similarity index 72% rename from docs/reference/repositories-metering/apis/get-repositories-metrics.asciidoc rename to docs/reference/repositories-metering/apis/get-repositories-metering.asciidoc index dd1aa2aae5cde..005bbdfec9b68 100644 --- a/docs/reference/repositories-metering/apis/get-repositories-metrics.asciidoc +++ b/docs/reference/repositories-metering/apis/get-repositories-metering.asciidoc @@ -1,34 +1,35 @@ -[[get-repositories-metrics-api]] -=== Repositories metrics API +[role="xpack"] +[testenv="basic"] +[[get-repositories-metering-api]] +=== Repositories metering API ++++ -Repositories metrics API +Repositories metering API ++++ -Returns cluster repositories metrics. +Returns cluster repositories metering. -[[get-repositories-metrics-api-request]] +[[get-repositories-metering-api-request]] ==== {api-request-title} -`GET /_nodes//_repositories_metrics` +`GET /_nodes//_repositories_metering` -[[get-repositories-metrics-api-desc]] +[[get-repositories-metering-api-desc]] ==== {api-description-title} -You can use the cluster repositories metrics API to retrieve repositories metrics in a cluster. - +You can use the cluster repositories metering API to retrieve repositories metrics in a cluster. All the nodes selective options are explained <>. -[[get-repositories-metrics-api-path-params]] +[[get-repositories-metering-api-path-params]] ==== {api-path-parms-title} include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] -[[get-repositories-metrics-api-query-params]] +[[get-repositories-metering-api-query-params]] ==== {api-query-parms-title} [role="child_attributes"] -[[get-repositories-metrics-api-response-body]] +[[get-repositories-metering-api-response-body]] ==== {api-response-body-title} `_nodes`:: @@ -105,7 +106,8 @@ A flag that tells whether or not this object has been archived. (long) The cluster state version when this object was created, this field can be used as a logical timestamp to delete all the archived metrics up -to a observed cluster version. +to a observed cluster version. This field is only present for archived +objects. `request_counts`:: (object) @@ -115,37 +117,44 @@ grouped by request type. .Properties of `request_counts` for repository type `Azure` [%collapsible%open] ====== -`HEAD`:: +Azure storage https://azure.microsoft.com/en-us/pricing/details/storage/blobs/[pricing]. +`GetBlobProperties`:: (long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-properties[Get Blob Properties] requests. -`GET`:: +`GetBlob`:: (long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob[Get Blob] requests. -`PUT`:: +`ListBlobs`:: +(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/list-blobs[List Blobs] requests. +`PutBlob`:: (long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob[Put Blob] requests. +`PutBlobBlock`:: +(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-block[Put Block] requests. ====== + .Properties of `request_counts` for repository type `GCP` [%collapsible%open] ====== -`GET`:: +Google Cloud Platform storage https://cloud.google.com/storage/pricing[pricing]. +`storage.objects.get`:: (long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/get[GET] requests. -`LIST`:: +`storage.objects.list`:: (long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/list[LIST] requests. -`PUT`:: +`storage.objects.resumable-insert`:: (long) Number of https://cloud.google.com/storage/docs/performing-resumable-uploads[PUT] requests. -`POST`:: +`storage.objects.insert`:: (long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/insert[POST] requests. ====== + .Properties of `request_counts` for repository type `S3` [%collapsible%open] ====== -`GET`:: +Amazon Web Services Simple Storage Service https://aws.amazon.com/s3/pricing/[pricing]. +`GetObject`:: (long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html[GET] requests. -`LIST`:: +`ListObjects`:: (long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html[LIST] requests. -`PUT`:: +`PutObject`:: (long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html[PUT] requests. -`POST`:: +`PutMultipartObject`:: (long) Number of https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html[MultiPart] requests. ====== ===== diff --git a/docs/reference/repositories-metering/repositories-metering.asciidoc b/docs/reference/repositories-metering/repositories-metering.asciidoc index 52bdc53480762..d3477d15a03c4 100644 --- a/docs/reference/repositories-metering/repositories-metering.asciidoc +++ b/docs/reference/repositories-metering/repositories-metering.asciidoc @@ -1,5 +1,5 @@ [role="xpack"] -[testenv="enterprise"] +[testenv="basic"] [[repositories-metering-apis]] == Repositories metering APIs @@ -8,8 +8,8 @@ experimental[] You can use the following APIs to retrieve repositories metrics. This is an API meant to be used by Elastic's commercial offerings. -* <> -* <> +* <> +* <> -include::get-repositories-metrics.asciidoc[] -include::clear-repositories-metrics-archive.asciidoc[] +include::get-repositories-metering.asciidoc[] +include::clear-repositories-metering-archive.asciidoc[] diff --git a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java index 7896af44a40e1..88e5f99d4a605 100644 --- a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -181,15 +181,15 @@ private AzureHTTPStatsCollectorHandler(HttpHandler delegate) { @Override protected void maybeTrack(String request, Headers headers) { if (Regex.simpleMatch("GET /*/*", request)) { - trackRequest("GET"); + trackRequest("GetBlob"); } else if (Regex.simpleMatch("HEAD /*/*", request)) { - trackRequest("HEAD"); + trackRequest("GetBlobProperties"); } else if (listPattern.test(request)) { - trackRequest("LIST"); + trackRequest("ListBlobs"); } else if (isBlockUpload(request)) { - trackRequest("PUT_BLOCK"); + trackRequest("PutBlock"); } else if (Regex.simpleMatch("PUT /*/*", request)) { - trackRequest("PUT"); + trackRequest("PutBlob"); } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java index 6f59f94dad8c5..6521fb2d6eae5 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java @@ -384,11 +384,11 @@ private static class Stats { private final AtomicLong putBlockOperations = new AtomicLong(); private Map toMap() { - return Map.of("GET", getOperations.get(), - "LIST", listOperations.get(), - "HEAD", headOperations.get(), - "PUT", putOperations.get(), - "PUT_BLOCK", putBlockOperations.get()); + return Map.of("GetBlob", getOperations.get(), + "ListBlobs", listOperations.get(), + "GetBlobProperties", headOperations.get(), + "PutBlob", putOperations.get(), + "PutBlock", putBlockOperations.get()); } } } diff --git a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index f73a8668fd82c..f65be55f21553 100644 --- a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -310,15 +310,15 @@ private static class GoogleCloudStorageStatsCollectorHttpHandler extends HttpSta @Override public void maybeTrack(final String request, Headers requestHeaders) { if (Regex.simpleMatch("GET /storage/v1/b/*/o/*", request)) { - trackRequest("GET"); + trackRequest("storage.objects.get"); } else if (Regex.simpleMatch("GET /storage/v1/b/*/o*", request)) { - trackRequest("LIST"); + trackRequest("storage.objects.list"); } else if (Regex.simpleMatch("GET /download/storage/v1/b/*", request)) { - trackRequest("GET"); + trackRequest("storage.objects.get"); } else if (Regex.simpleMatch("PUT /upload/storage/v1/b/*", request) && isLastPart(requestHeaders)) { - trackRequest("PUT"); + trackRequest("storage.objects.resumable-insert"); } else if (Regex.simpleMatch("POST /upload/storage/v1/b/*uploadType=multipart*", request)) { - trackRequest("POST"); + trackRequest("storage.objects.insert"); } } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java index 4a0de249a3e78..d0c83f50bceb4 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java @@ -58,10 +58,10 @@ String getTrackedBucket() { Map toMap() { final Map results = new HashMap<>(); - results.put("GET", getCount.get()); - results.put("LIST", listCount.get()); - results.put("PUT", putCount.get()); - results.put("POST", postCount.get()); + results.put("storage.objects.get", getCount.get()); + results.put("storage.objects.list", listCount.get()); + results.put("storage.objects.resumable-insert", putCount.get()); + results.put("storage.objects.insert", postCount.get()); return results; } } diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 3bf452cdb949b..2a8b7dafd70af 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -287,13 +287,13 @@ private static class S3StatsCollectorHttpHandler extends HttpStatsCollectorHandl @Override public void maybeTrack(final String request, Headers requestHeaders) { if (Regex.simpleMatch("GET /*/?prefix=*", request)) { - trackRequest("LIST"); + trackRequest("ListObjects"); } else if (Regex.simpleMatch("GET /*/*", request)) { - trackRequest("GET"); + trackRequest("GetObject"); } else if (isMultiPartUpload(request)) { - trackRequest("POST"); + trackRequest("PutMultipartObject"); } else if (Regex.simpleMatch("PUT /*/*", request)) { - trackRequest("PUT"); + trackRequest("PutObject"); } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 10c0c0635e455..c3659a09fdaf4 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -204,10 +204,10 @@ static class Stats { Map toMap() { final Map results = new HashMap<>(); - results.put("GET", getCount.get()); - results.put("LIST", listCount.get()); - results.put("PUT", putCount.get()); - results.put("POST", postCount.get()); + results.put("GetObject", getCount.get()); + results.put("ListObjects", listCount.get()); + results.put("PutObject", putCount.get()); + results.put("PutMultipartObject", postCount.get()); return results; } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index d8c16cb98c0bb..4da9ecfcf7b8d 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -90,7 +90,6 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C private volatile Map repositories = Collections.emptyMap(); private final RepositoriesStatsArchive repositoriesStatsArchive; - private volatile long lastKnownClusterVersion; public RepositoriesService(Settings settings, ClusterService clusterService, TransportService transportService, Map typesRegistry, Map internalTypesRegistry, @@ -315,7 +314,6 @@ static boolean isDedicatedVotingOnlyNode(Set roles) { public void applyClusterState(ClusterChangedEvent event) { try { final ClusterState state = event.state(); - this.lastKnownClusterVersion = state.version(); RepositoriesMetadata oldMetadata = event.previousState().getMetadata().custom(RepositoriesMetadata.TYPE); RepositoriesMetadata newMetadata = state.getMetadata().custom(RepositoriesMetadata.TYPE); @@ -439,7 +437,7 @@ private List getRepositoryStatsForActiveRepositories() return Stream.concat(repositories.values().stream(), internalRepositories.values().stream()) .filter(r -> r instanceof MeteredBlobStoreRepository) .map(r -> (MeteredBlobStoreRepository) r) - .map(r -> r.statsSnapshot(lastKnownClusterVersion)) + .map(MeteredBlobStoreRepository::statsSnapshot) .collect(Collectors.toList()); } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java index 3b805d01d31ca..df42d76d80de6 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java @@ -30,6 +30,7 @@ import java.util.Objects; public final class RepositoryStatsSnapshot implements Writeable, ToXContent { + public static final long UNKNOWN_CLUSTER_VERSION = -1; private final RepositoryInfo repositoryInfo; private final RepositoryStats repositoryStats; private final long clusterVersion; @@ -39,6 +40,7 @@ public RepositoryStatsSnapshot(RepositoryInfo repositoryInfo, RepositoryStats repositoryStats, long clusterVersion, boolean archived) { + assert archived != (clusterVersion == UNKNOWN_CLUSTER_VERSION); this.repositoryInfo = repositoryInfo; this.repositoryStats = repositoryStats; this.clusterVersion = clusterVersion; @@ -80,9 +82,11 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); repositoryInfo.toXContent(builder, params); - builder.field("cluster_version", clusterVersion); builder.field("request_counts", repositoryStats.requestCounts); builder.field("archived", archived); + if (archived) { + builder.field("cluster_version", clusterVersion); + } builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java index 2a1ae0e525f98..bf19ad90c745c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java @@ -47,8 +47,8 @@ public MeteredBlobStoreRepository(RepositoryMetadata metadata, threadPool.absoluteTimeInMillis()); } - public RepositoryStatsSnapshot statsSnapshot(long clusterVersion) { - return new RepositoryStatsSnapshot(repositoryInfo, stats(), clusterVersion, false); + public RepositoryStatsSnapshot statsSnapshot() { + return new RepositoryStatsSnapshot(repositoryInfo, stats(), RepositoryStatsSnapshot.UNKNOWN_CLUSTER_VERSION, false); } public RepositoryStatsSnapshot statsSnapshotForArchival(long clusterVersion) { diff --git a/x-pack/plugin/repositories-metering-api/build.gradle b/x-pack/plugin/repositories-metering-api/build.gradle index 998128d168a3b..23c911c15c760 100644 --- a/x-pack/plugin/repositories-metering-api/build.gradle +++ b/x-pack/plugin/repositories-metering-api/build.gradle @@ -4,7 +4,7 @@ apply plugin: 'elasticsearch.esplugin' esplugin { name 'repositories-metering-api' description 'Repositories metering API' - classname 'org.elasticsearch.xpack.repositories.metrics.RepositoriesMeteringPlugin' + classname 'org.elasticsearch.xpack.repositories.metering.RepositoriesMeteringPlugin' extendedPlugins = ['x-pack-core'] } archivesBaseName = 'x-pack-repositories-metering-api' diff --git a/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metrics/azure/AzureRepositoriesMetricsIT.java b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java similarity index 83% rename from x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metrics/azure/AzureRepositoriesMetricsIT.java rename to x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java index 37799f195130f..37ff40e31ac62 100644 --- a/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metrics/azure/AzureRepositoriesMetricsIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java @@ -3,14 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.azure; +package org.elasticsearch.xpack.repositories.metering.azure; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.repositories.metrics.AbstractRepositoriesMetricsAPIRestTestCase; +import org.elasticsearch.xpack.repositories.metering.AbstractRepositoriesMeteringAPIRestTestCase; import java.util.List; -public class AzureRepositoriesMetricsIT extends AbstractRepositoriesMetricsAPIRestTestCase { +public class AzureRepositoriesMeteringIT extends AbstractRepositoriesMeteringAPIRestTestCase { @Override protected String repositoryType() { diff --git a/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metrics/gcs/GCSRepositoriesMetricsIT.java b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java similarity index 83% rename from x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metrics/gcs/GCSRepositoriesMetricsIT.java rename to x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java index 51160607ed914..f0104ae767467 100644 --- a/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metrics/gcs/GCSRepositoriesMetricsIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java @@ -3,14 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.gcs; +package org.elasticsearch.xpack.repositories.metering.gcs; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.repositories.metrics.AbstractRepositoriesMetricsAPIRestTestCase; +import org.elasticsearch.xpack.repositories.metering.AbstractRepositoriesMeteringAPIRestTestCase; import java.util.List; -public class GCSRepositoriesMetricsIT extends AbstractRepositoriesMetricsAPIRestTestCase { +public class GCSRepositoriesMeteringIT extends AbstractRepositoriesMeteringAPIRestTestCase { @Override protected String repositoryType() { diff --git a/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metrics/s3/S3RepositoriesMetricsIT.java b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java similarity index 84% rename from x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metrics/s3/S3RepositoriesMetricsIT.java rename to x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java index 6e548c4df25e1..414163cfeee3a 100644 --- a/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metrics/s3/S3RepositoriesMetricsIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java @@ -3,14 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.s3; +package org.elasticsearch.xpack.repositories.metering.s3; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.repositories.metrics.AbstractRepositoriesMetricsAPIRestTestCase; +import org.elasticsearch.xpack.repositories.metering.AbstractRepositoriesMeteringAPIRestTestCase; import java.util.List; -public class S3RepositoriesMetricsIT extends AbstractRepositoriesMetricsAPIRestTestCase { +public class S3RepositoriesMeteringIT extends AbstractRepositoriesMeteringAPIRestTestCase { @Override protected String repositoryType() { diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/RepositoriesMeteringPlugin.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/RepositoriesMeteringPlugin.java similarity index 60% rename from x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/RepositoriesMeteringPlugin.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/RepositoriesMeteringPlugin.java index a687885772fab..f7263286e3691 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/RepositoriesMeteringPlugin.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/RepositoriesMeteringPlugin.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics; +package org.elasticsearch.xpack.repositories.metering; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -18,12 +18,12 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.xpack.repositories.metrics.action.ClearRepositoriesMetricsArchiveAction; -import org.elasticsearch.xpack.repositories.metrics.action.RepositoriesMetricsAction; -import org.elasticsearch.xpack.repositories.metrics.action.TransportClearRepositoriesStatsArchiveAction; -import org.elasticsearch.xpack.repositories.metrics.action.TransportRepositoriesStatsAction; -import org.elasticsearch.xpack.repositories.metrics.rest.RestClearRepositoriesMetricsArchiveAction; -import org.elasticsearch.xpack.repositories.metrics.rest.RestGetRepositoriesMetricsAction; +import org.elasticsearch.xpack.repositories.metering.action.ClearRepositoriesMeteringArchiveAction; +import org.elasticsearch.xpack.repositories.metering.action.RepositoriesMeteringAction; +import org.elasticsearch.xpack.repositories.metering.action.TransportClearRepositoriesStatsArchiveAction; +import org.elasticsearch.xpack.repositories.metering.action.TransportRepositoriesStatsAction; +import org.elasticsearch.xpack.repositories.metering.rest.RestClearRepositoriesMeteringArchiveAction; +import org.elasticsearch.xpack.repositories.metering.rest.RestGetRepositoriesMeteringAction; import java.util.List; import java.util.function.Supplier; @@ -33,8 +33,8 @@ public final class RepositoriesMeteringPlugin extends Plugin implements ActionPl @Override public List> getActions() { return List.of( - new ActionHandler<>(RepositoriesMetricsAction.INSTANCE, TransportRepositoriesStatsAction.class), - new ActionHandler<>(ClearRepositoriesMetricsArchiveAction.INSTANCE, TransportClearRepositoriesStatsArchiveAction.class) + new ActionHandler<>(RepositoriesMeteringAction.INSTANCE, TransportRepositoriesStatsAction.class), + new ActionHandler<>(ClearRepositoriesMeteringArchiveAction.INSTANCE, TransportClearRepositoriesStatsArchiveAction.class) ); } @@ -48,6 +48,6 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - return List.of(new RestGetRepositoriesMetricsAction(), new RestClearRepositoriesMetricsArchiveAction()); + return List.of(new RestGetRepositoriesMeteringAction(), new RestClearRepositoriesMeteringArchiveAction()); } } diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveAction.java new file mode 100644 index 0000000000000..4ddddc198e454 --- /dev/null +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveAction.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.metering.action; + +import org.elasticsearch.action.ActionType; + +public final class ClearRepositoriesMeteringArchiveAction extends ActionType { + public static final ClearRepositoriesMeteringArchiveAction INSTANCE = new ClearRepositoriesMeteringArchiveAction(); + + static final String NAME = "cluster:monitor/xpack/repositories_metering/clear_metering_archive"; + + ClearRepositoriesMeteringArchiveAction() { + super(NAME, RepositoriesMeteringResponse::new); + } +} diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveRequest.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveRequest.java similarity index 70% rename from x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveRequest.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveRequest.java index 43ec61700160d..98f0a8ec0a6c2 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveRequest.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveRequest.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.action; +package org.elasticsearch.xpack.repositories.metering.action; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; @@ -12,15 +12,15 @@ import java.io.IOException; -public final class ClearRepositoriesMetricsArchiveRequest extends BaseNodesRequest { +public final class ClearRepositoriesMeteringArchiveRequest extends BaseNodesRequest { private final long maxVersionToClear; - public ClearRepositoriesMetricsArchiveRequest(StreamInput in) throws IOException { + public ClearRepositoriesMeteringArchiveRequest(StreamInput in) throws IOException { super(in); this.maxVersionToClear = in.readLong(); } - public ClearRepositoriesMetricsArchiveRequest(long maxVersionToClear, String... nodesIds) { + public ClearRepositoriesMeteringArchiveRequest(long maxVersionToClear, String... nodesIds) { super(nodesIds); this.maxVersionToClear = maxVersionToClear; } diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringAction.java new file mode 100644 index 0000000000000..9d7cd8bbe7107 --- /dev/null +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringAction.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.repositories.metering.action; + +import org.elasticsearch.action.ActionType; + +public final class RepositoriesMeteringAction extends ActionType { + public static final RepositoriesMeteringAction INSTANCE = new RepositoriesMeteringAction(); + + static final String NAME = "cluster:monitor/xpack/repositories_metering/get_metrics"; + + RepositoriesMeteringAction() { + super(NAME, RepositoriesMeteringResponse::new); + } +} diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsRequest.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringRequest.java similarity index 59% rename from x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsRequest.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringRequest.java index f887a8c27db0d..92119da4f0027 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsRequest.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringRequest.java @@ -4,19 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.action; +package org.elasticsearch.xpack.repositories.metering.action; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import java.io.IOException; -public final class RepositoriesMetricsRequest extends BaseNodesRequest { - public RepositoriesMetricsRequest(StreamInput in) throws IOException { +public final class RepositoriesMeteringRequest extends BaseNodesRequest { + public RepositoriesMeteringRequest(StreamInput in) throws IOException { super(in); } - public RepositoriesMetricsRequest(String... nodesIds) { + public RepositoriesMeteringRequest(String... nodesIds) { super(nodesIds); } } diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponse.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponse.java similarity index 65% rename from x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponse.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponse.java index 650ae0a158115..382066c06b695 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponse.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponse.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.action; +package org.elasticsearch.xpack.repositories.metering.action; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.nodes.BaseNodesResponse; @@ -17,34 +17,34 @@ import java.io.IOException; import java.util.List; -public final class RepositoriesMetricsResponse extends BaseNodesResponse implements ToXContentFragment { +public final class RepositoriesMeteringResponse extends BaseNodesResponse implements ToXContentFragment { - public RepositoriesMetricsResponse(StreamInput in) throws IOException { + public RepositoriesMeteringResponse(StreamInput in) throws IOException { super(in); } - public RepositoriesMetricsResponse( + public RepositoriesMeteringResponse( ClusterName clusterName, - List nodes, + List nodes, List failures ) { super(clusterName, nodes, failures); } @Override - protected List readNodesFrom(StreamInput in) throws IOException { - return in.readList(RepositoriesNodeMetricsResponse::new); + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(RepositoriesNodeMeteringResponse::new); } @Override - protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { out.writeList(nodes); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("nodes"); - for (RepositoriesNodeMetricsResponse nodeStats : getNodes()) { + for (RepositoriesNodeMeteringResponse nodeStats : getNodes()) { nodeStats.toXContent(builder, params); } builder.endObject(); diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesNodeMetricsResponse.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesNodeMeteringResponse.java similarity index 80% rename from x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesNodeMetricsResponse.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesNodeMeteringResponse.java index 6f08becc7a656..cf463e6f9b22a 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesNodeMetricsResponse.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesNodeMeteringResponse.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.action; +package org.elasticsearch.xpack.repositories.metering.action; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -18,16 +18,16 @@ import java.io.IOException; import java.util.List; -public final class RepositoriesNodeMetricsResponse extends BaseNodeResponse implements ToXContentFragment { +public final class RepositoriesNodeMeteringResponse extends BaseNodeResponse implements ToXContentFragment { final List repositoryStatsSnapshots; - public RepositoriesNodeMetricsResponse(DiscoveryNode node, List repositoryStatsSnapshots) { + public RepositoriesNodeMeteringResponse(DiscoveryNode node, List repositoryStatsSnapshots) { super(node); this.repositoryStatsSnapshots = repositoryStatsSnapshots; } - public RepositoriesNodeMetricsResponse(StreamInput in) throws IOException { + public RepositoriesNodeMeteringResponse(StreamInput in) throws IOException { super(in); this.repositoryStatsSnapshots = in.readList(RepositoryStatsSnapshot::new); } diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java similarity index 72% rename from x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportClearRepositoriesStatsArchiveAction.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java index 6084c80d973f5..2f020a0d1bee9 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.action; +package org.elasticsearch.xpack.repositories.metering.action; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; @@ -24,10 +24,10 @@ import java.util.List; public final class TransportClearRepositoriesStatsArchiveAction extends TransportNodesAction< - ClearRepositoriesMetricsArchiveRequest, - RepositoriesMetricsResponse, + ClearRepositoriesMeteringArchiveRequest, + RepositoriesMeteringResponse, TransportClearRepositoriesStatsArchiveAction.ClearRepositoriesStatsArchiveNodeRequest, - RepositoriesNodeMetricsResponse> { + RepositoriesNodeMeteringResponse> { private final RepositoriesService repositoriesService; @@ -40,42 +40,42 @@ public TransportClearRepositoriesStatsArchiveAction( RepositoriesService repositoriesService ) { super( - ClearRepositoriesMetricsArchiveAction.NAME, + ClearRepositoriesMeteringArchiveAction.NAME, threadPool, clusterService, transportService, actionFilters, - ClearRepositoriesMetricsArchiveRequest::new, + ClearRepositoriesMeteringArchiveRequest::new, ClearRepositoriesStatsArchiveNodeRequest::new, ThreadPool.Names.SAME, - RepositoriesNodeMetricsResponse.class + RepositoriesNodeMeteringResponse.class ); this.repositoriesService = repositoriesService; } @Override - protected RepositoriesMetricsResponse newResponse( - ClearRepositoriesMetricsArchiveRequest request, - List nodesResponses, + protected RepositoriesMeteringResponse newResponse( + ClearRepositoriesMeteringArchiveRequest request, + List nodesResponses, List failures ) { - return new RepositoriesMetricsResponse(clusterService.getClusterName(), nodesResponses, failures); + return new RepositoriesMeteringResponse(clusterService.getClusterName(), nodesResponses, failures); } @Override - protected ClearRepositoriesStatsArchiveNodeRequest newNodeRequest(ClearRepositoriesMetricsArchiveRequest request) { + protected ClearRepositoriesStatsArchiveNodeRequest newNodeRequest(ClearRepositoriesMeteringArchiveRequest request) { return new ClearRepositoriesStatsArchiveNodeRequest(request.getMaxVersionToClear()); } @Override - protected RepositoriesNodeMetricsResponse newNodeResponse(StreamInput in) throws IOException { - return new RepositoriesNodeMetricsResponse(in); + protected RepositoriesNodeMeteringResponse newNodeResponse(StreamInput in) throws IOException { + return new RepositoriesNodeMeteringResponse(in); } @Override - protected RepositoriesNodeMetricsResponse nodeOperation(ClearRepositoriesStatsArchiveNodeRequest request, Task task) { + protected RepositoriesNodeMeteringResponse nodeOperation(ClearRepositoriesStatsArchiveNodeRequest request, Task task) { List clearedStats = repositoriesService.clearRepositoriesStatsArchive(request.maxVersionToClear); - return new RepositoriesNodeMetricsResponse(clusterService.localNode(), clearedStats); + return new RepositoriesNodeMeteringResponse(clusterService.localNode(), clearedStats); } static final class ClearRepositoriesStatsArchiveNodeRequest extends TransportRequest { diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportRepositoriesStatsAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java similarity index 67% rename from x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportRepositoriesStatsAction.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java index 7e0092dcb626b..74541a70a5eeb 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/TransportRepositoriesStatsAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.action; +package org.elasticsearch.xpack.repositories.metering.action; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; @@ -22,10 +22,10 @@ import java.util.List; public final class TransportRepositoriesStatsAction extends TransportNodesAction< - RepositoriesMetricsRequest, - RepositoriesMetricsResponse, + RepositoriesMeteringRequest, + RepositoriesMeteringResponse, TransportRepositoriesStatsAction.RepositoriesNodeStatsRequest, - RepositoriesNodeMetricsResponse> { + RepositoriesNodeMeteringResponse> { private final RepositoriesService repositoriesService; @@ -38,41 +38,41 @@ public TransportRepositoriesStatsAction( RepositoriesService repositoriesService ) { super( - RepositoriesMetricsAction.NAME, + RepositoriesMeteringAction.NAME, threadPool, clusterService, transportService, actionFilters, - RepositoriesMetricsRequest::new, + RepositoriesMeteringRequest::new, RepositoriesNodeStatsRequest::new, ThreadPool.Names.SAME, - RepositoriesNodeMetricsResponse.class + RepositoriesNodeMeteringResponse.class ); this.repositoriesService = repositoriesService; } @Override - protected RepositoriesMetricsResponse newResponse( - RepositoriesMetricsRequest request, - List repositoriesNodeStatsResponses, + protected RepositoriesMeteringResponse newResponse( + RepositoriesMeteringRequest request, + List repositoriesNodeStatsResponses, List failures ) { - return new RepositoriesMetricsResponse(clusterService.getClusterName(), repositoriesNodeStatsResponses, failures); + return new RepositoriesMeteringResponse(clusterService.getClusterName(), repositoriesNodeStatsResponses, failures); } @Override - protected RepositoriesNodeStatsRequest newNodeRequest(RepositoriesMetricsRequest request) { + protected RepositoriesNodeStatsRequest newNodeRequest(RepositoriesMeteringRequest request) { return new RepositoriesNodeStatsRequest(); } @Override - protected RepositoriesNodeMetricsResponse newNodeResponse(StreamInput in) throws IOException { - return new RepositoriesNodeMetricsResponse(in); + protected RepositoriesNodeMeteringResponse newNodeResponse(StreamInput in) throws IOException { + return new RepositoriesNodeMeteringResponse(in); } @Override - protected RepositoriesNodeMetricsResponse nodeOperation(RepositoriesNodeStatsRequest request, Task task) { - return new RepositoriesNodeMetricsResponse(clusterService.localNode(), repositoriesService.repositoriesStats()); + protected RepositoriesNodeMeteringResponse nodeOperation(RepositoriesNodeStatsRequest request, Task task) { + return new RepositoriesNodeMeteringResponse(clusterService.localNode(), repositoriesService.repositoriesStats()); } static final class RepositoriesNodeStatsRequest extends TransportRequest { diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/rest/RestClearRepositoriesMeteringArchiveAction.java similarity index 67% rename from x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/rest/RestClearRepositoriesMeteringArchiveAction.java index ab38859ae4294..ddf15ae1c46a3 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestClearRepositoriesMetricsArchiveAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/rest/RestClearRepositoriesMeteringArchiveAction.java @@ -4,19 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.rest; +package org.elasticsearch.xpack.repositories.metering.rest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; -import org.elasticsearch.xpack.repositories.metrics.action.ClearRepositoriesMetricsArchiveAction; -import org.elasticsearch.xpack.repositories.metrics.action.ClearRepositoriesMetricsArchiveRequest; +import org.elasticsearch.xpack.repositories.metering.action.ClearRepositoriesMeteringArchiveAction; +import org.elasticsearch.xpack.repositories.metering.action.ClearRepositoriesMeteringArchiveRequest; import java.util.List; -public class RestClearRepositoriesMetricsArchiveAction extends BaseRestHandler { +public class RestClearRepositoriesMeteringArchiveAction extends BaseRestHandler { @Override public String getName() { return "clear_repositories_metrics_archive_action"; @@ -24,19 +24,19 @@ public String getName() { @Override public List routes() { - return List.of(new Route(RestRequest.Method.DELETE, "/_nodes/{nodeId}/_repositories_metrics/{maxVersionToClear}")); + return List.of(new Route(RestRequest.Method.DELETE, "/_nodes/{nodeId}/_repositories_metering/{maxVersionToClear}")); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); long maxVersionToClear = request.paramAsLong("maxVersionToClear", -1); - ClearRepositoriesMetricsArchiveRequest clearArchivesRequest = new ClearRepositoriesMetricsArchiveRequest( + ClearRepositoriesMeteringArchiveRequest clearArchivesRequest = new ClearRepositoriesMeteringArchiveRequest( maxVersionToClear, nodesIds ); return channel -> client.execute( - ClearRepositoriesMetricsArchiveAction.INSTANCE, + ClearRepositoriesMeteringArchiveAction.INSTANCE, clearArchivesRequest, new RestActions.NodesResponseRestListener<>(channel) ); diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestGetRepositoriesMetricsAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/rest/RestGetRepositoriesMeteringAction.java similarity index 62% rename from x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestGetRepositoriesMetricsAction.java rename to x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/rest/RestGetRepositoriesMeteringAction.java index 67684b4ba49a6..4d3bfd49c7acd 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/rest/RestGetRepositoriesMetricsAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/rest/RestGetRepositoriesMeteringAction.java @@ -4,37 +4,37 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.rest; +package org.elasticsearch.xpack.repositories.metering.rest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; -import org.elasticsearch.xpack.repositories.metrics.action.RepositoriesMetricsRequest; -import org.elasticsearch.xpack.repositories.metrics.action.RepositoriesMetricsAction; +import org.elasticsearch.xpack.repositories.metering.action.RepositoriesMeteringRequest; +import org.elasticsearch.xpack.repositories.metering.action.RepositoriesMeteringAction; import java.util.List; -public final class RestGetRepositoriesMetricsAction extends BaseRestHandler { +public final class RestGetRepositoriesMeteringAction extends BaseRestHandler { @Override public String getName() { - return "get_repositories_metrics_action"; + return "get_repositories_metering_action"; } @Override public List routes() { - return List.of(new Route(RestRequest.Method.GET, "/_nodes/{nodeId}/_repositories_metrics")); + return List.of(new Route(RestRequest.Method.GET, "/_nodes/{nodeId}/_repositories_metering")); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); - RepositoriesMetricsRequest repositoriesMetricsRequest = new RepositoriesMetricsRequest(nodesIds); + RepositoriesMeteringRequest repositoriesMeteringRequest = new RepositoriesMeteringRequest(nodesIds); return channel -> client.execute( - RepositoriesMetricsAction.INSTANCE, - repositoriesMetricsRequest, + RepositoriesMeteringAction.INSTANCE, + repositoriesMeteringRequest, new RestActions.NodesResponseRestListener<>(channel) ); } diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveAction.java deleted file mode 100644 index f1bef39073a0f..0000000000000 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/ClearRepositoriesMetricsArchiveAction.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.repositories.metrics.action; - -import org.elasticsearch.action.ActionType; - -public final class ClearRepositoriesMetricsArchiveAction extends ActionType { - public static final ClearRepositoriesMetricsArchiveAction INSTANCE = new ClearRepositoriesMetricsArchiveAction(); - - static final String NAME = "cluster:monitor/xpack/repositories_metrics/clear_repositories_metrics_archive"; - - ClearRepositoriesMetricsArchiveAction() { - super(NAME, RepositoriesMetricsResponse::new); - } -} diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsAction.java deleted file mode 100644 index f8850c361e88d..0000000000000 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsAction.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.repositories.metrics.action; - -import org.elasticsearch.action.ActionType; - -public final class RepositoriesMetricsAction extends ActionType { - public static final RepositoriesMetricsAction INSTANCE = new RepositoriesMetricsAction(); - - static final String NAME = "cluster:monitor/xpack/repositories_metrics/get_metrics"; - - RepositoriesMetricsAction() { - super(NAME, RepositoriesMetricsResponse::new); - } -} diff --git a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/AbstractRepositoriesMetricsAPIRestTestCase.java b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/AbstractRepositoriesMeteringAPIRestTestCase.java similarity index 97% rename from x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/AbstractRepositoriesMetricsAPIRestTestCase.java rename to x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/AbstractRepositoriesMeteringAPIRestTestCase.java index 7c90acef78a0c..bfad5d87f00db 100644 --- a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/AbstractRepositoriesMetricsAPIRestTestCase.java +++ b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/AbstractRepositoriesMeteringAPIRestTestCase.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics; +package org.elasticsearch.xpack.repositories.metering; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpPost; @@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; -public abstract class AbstractRepositoriesMetricsAPIRestTestCase extends ESRestTestCase { +public abstract class AbstractRepositoriesMeteringAPIRestTestCase extends ESRestTestCase { protected abstract String repositoryType(); protected abstract String repositoryLocation(); @@ -299,7 +299,7 @@ private void assertAllRequestCountsAreZero(RepositoryStatsSnapshot statsSnapshot } private List getRepositoriesStats() throws IOException { - Map response = getAsMap("/_nodes/_all/_repositories_metrics"); + Map response = getAsMap("/_nodes/_all/_repositories_metering"); return parseRepositoriesStatsResponse(response); } @@ -314,8 +314,11 @@ private List parseRepositoriesStatsResponse(Map nodeStatSnapshot : nodeStats) { RepositoryInfo repositoryInfo = parseRepositoryInfo(nodeStatSnapshot); Map intRequestCounters = extractValue(nodeStatSnapshot, "request_counts"); - int clusterVersion = extractValue(nodeStatSnapshot, "cluster_version"); boolean archived = extractValue(nodeStatSnapshot, "archived"); + int clusterVersion = (int) RepositoryStatsSnapshot.UNKNOWN_CLUSTER_VERSION; + if (archived) { + clusterVersion = extractValue(nodeStatSnapshot, "cluster_version"); + } Map requestCounters = intRequestCounters.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().longValue())); @@ -348,7 +351,7 @@ private Set getNodeIds() throws IOException { } private List clearRepositoriesStats(long maxVersionToClear) throws IOException { - final Request request = new Request(HttpDelete.METHOD_NAME, "/_nodes/_all/_repositories_metrics/" + maxVersionToClear); + final Request request = new Request(HttpDelete.METHOD_NAME, "/_nodes/_all/_repositories_metering/" + maxVersionToClear); final Response response = client().performRequest(request); assertThat( "Failed to clear repositories stats: " + response, diff --git a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponseTests.java b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java similarity index 71% rename from x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponseTests.java rename to x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java index 19b5dea0e31a5..354ba5dfab4ed 100644 --- a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metrics/action/RepositoriesMetricsResponseTests.java +++ b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.repositories.metrics.action; +package org.elasticsearch.xpack.repositories.metering.action; import org.elasticsearch.Version; import org.elasticsearch.action.FailedNodeException; @@ -22,25 +22,25 @@ import static org.hamcrest.Matchers.equalTo; -public class RepositoriesMetricsResponseTests extends ESTestCase { +public class RepositoriesMeteringResponseTests extends ESTestCase { public void testSerializationRoundtrip() throws Exception { - final RepositoriesMetricsResponse repositoriesMetricsResponse = createResponse(); - final RepositoriesMetricsResponse deserializedResponse = copyWriteable( - repositoriesMetricsResponse, + final RepositoriesMeteringResponse repositoriesMeteringResponse = createResponse(); + final RepositoriesMeteringResponse deserializedResponse = copyWriteable( + repositoriesMeteringResponse, writableRegistry(), - RepositoriesMetricsResponse::new, + RepositoriesMeteringResponse::new, Version.CURRENT ); - assertResponsesAreEqual(repositoriesMetricsResponse, deserializedResponse); + assertResponsesAreEqual(repositoriesMeteringResponse, deserializedResponse); } - private void assertResponsesAreEqual(RepositoriesMetricsResponse response, RepositoriesMetricsResponse otherResponse) { - List nodeResponses = response.getNodes(); - List otherNodeResponses = otherResponse.getNodes(); + private void assertResponsesAreEqual(RepositoriesMeteringResponse response, RepositoriesMeteringResponse otherResponse) { + List nodeResponses = response.getNodes(); + List otherNodeResponses = otherResponse.getNodes(); assertThat(nodeResponses.size(), equalTo(otherNodeResponses.size())); for (int i = 0; i < nodeResponses.size(); i++) { - RepositoriesNodeMetricsResponse nodeResponse = nodeResponses.get(i); - RepositoriesNodeMetricsResponse otherNodeResponse = otherNodeResponses.get(i); + RepositoriesNodeMeteringResponse nodeResponse = nodeResponses.get(i); + RepositoriesNodeMeteringResponse otherNodeResponse = otherNodeResponses.get(i); assertThat(nodeResponse.repositoryStatsSnapshots, equalTo(otherNodeResponse.repositoryStatsSnapshots)); } @@ -55,10 +55,10 @@ private void assertResponsesAreEqual(RepositoriesMetricsResponse response, Repos } } - private RepositoriesMetricsResponse createResponse() { + private RepositoriesMeteringResponse createResponse() { ClusterName clusterName = new ClusterName("test"); int nodes = randomIntBetween(1, 10); - List nodeResponses = new ArrayList<>(nodes); + List nodeResponses = new ArrayList<>(nodes); for (int i = 0; i < nodes; i++) { DiscoveryNode node = new DiscoveryNode("nodeId" + i, buildNewFakeTransportAddress(), Version.CURRENT); int numberOfRepos = randomInt(10); @@ -72,16 +72,17 @@ private RepositoriesMetricsResponse createResponse() { long startedAt = System.currentTimeMillis() - 1; Long stoppedAt = randomBoolean() ? System.currentTimeMillis() : null; RepositoryInfo repositoryInfo = new RepositoryInfo(repoId, repoName, repoType, repoLocation, startedAt, stoppedAt); + boolean archived = randomBoolean(); RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot( repositoryInfo, new RepositoryStats(Map.of("GET", randomLongBetween(0, 2000))), - j, - randomBoolean() + archived ? j : RepositoryStatsSnapshot.UNKNOWN_CLUSTER_VERSION, + archived ); nodeRepoStats.add(statsSnapshot); } - nodeResponses.add(new RepositoriesNodeMetricsResponse(node, nodeRepoStats)); + nodeResponses.add(new RepositoriesNodeMeteringResponse(node, nodeRepoStats)); } int numberOfFailures = randomInt(20); @@ -95,6 +96,6 @@ private RepositoriesMetricsResponse createResponse() { failures.add(failedNodeException); } - return new RepositoriesMetricsResponse(clusterName, nodeResponses, failures); + return new RepositoriesMeteringResponse(clusterName, nodeResponses, failures); } } From ddc2cc19e54765697fa2be4994db79e9a3416075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 18 Aug 2020 17:09:35 +0200 Subject: [PATCH 11/19] Fix tests --- .../metering/azure/AzureRepositoriesMeteringIT.java | 4 ++-- .../repositories/metering/gcs/GCSRepositoriesMeteringIT.java | 4 ++-- .../repositories/metering/s3/S3RepositoriesMeteringIT.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java index 37ff40e31ac62..1e95dac5781dc 100644 --- a/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java @@ -38,11 +38,11 @@ protected Settings updatedRepositorySettings() { @Override protected List readCounterKeys() { - return List.of("GET", "HEAD", "LIST"); + return List.of("GetBlob", "GetBlobProperties", "ListBlobs"); } @Override protected List writeCounterKeys() { - return List.of("PUT"); + return List.of("PutBlob"); } } diff --git a/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java index f0104ae767467..63a031d921e8a 100644 --- a/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java @@ -37,11 +37,11 @@ protected Settings updatedRepositorySettings() { @Override protected List readCounterKeys() { - return List.of("GET", "LIST"); + return List.of("storage.objects.get", "storage.objects.list"); } @Override protected List writeCounterKeys() { - return List.of("POST"); + return List.of("storage.objects.insert"); } } diff --git a/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java index 414163cfeee3a..a5d7cb1a5c8d9 100644 --- a/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java @@ -38,11 +38,11 @@ protected Settings updatedRepositorySettings() { @Override protected List readCounterKeys() { - return List.of("GET", "LIST"); + return List.of("GetObject", "ListObjects"); } @Override protected List writeCounterKeys() { - return List.of("PUT"); + return List.of("PutObject"); } } From 1f93514257cf6d85adec4dd5d11040d8b17cb756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 18 Aug 2020 17:10:02 +0200 Subject: [PATCH 12/19] Improve docs --- ...ear-repositories-metering-archive.asciidoc | 22 +++---- .../apis/get-repositories-metering.asciidoc | 35 ++++++++++ .../repositories-meterings-body.asciidoc} | 64 ++++++------------- .../repositories-metering-apis.asciidoc} | 7 +- docs/reference/rest-api/index.asciidoc | 2 + 5 files changed, 69 insertions(+), 61 deletions(-) rename docs/reference/{repositories-metering => repositories-metering-api}/apis/clear-repositories-metering-archive.asciidoc (55%) create mode 100644 docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc rename docs/reference/{repositories-metering/apis/get-repositories-metering.asciidoc => repositories-metering-api/apis/repositories-meterings-body.asciidoc} (75%) rename docs/reference/{repositories-metering/repositories-metering.asciidoc => repositories-metering-api/repositories-metering-apis.asciidoc} (64%) diff --git a/docs/reference/repositories-metering/apis/clear-repositories-metering-archive.asciidoc b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc similarity index 55% rename from docs/reference/repositories-metering/apis/clear-repositories-metering-archive.asciidoc rename to docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc index c12c5c15ae452..7c9cc4553343b 100644 --- a/docs/reference/repositories-metering/apis/clear-repositories-metering-archive.asciidoc +++ b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc @@ -1,13 +1,12 @@ [role="xpack"] [testenv="basic"] [[clear-repositories-metering-archive-api]] -=== Clear repositories metering archive API +=== Clear repositories metering archive ++++ -Clear repositories metering archive API +Clear repositories metering archive ++++ -Removes the archived repositories metrics in the cluster up to -a particular version. +Removes the archived repositories meterings present in the cluster. [[clear-repositories-metering-archive-api-request]] ==== {api-request-title} @@ -17,10 +16,7 @@ a particular version. [[clear-repositories-metering-archive-api-desc]] ==== {api-description-title} -You can clear the archived repositories utilization metrics using this API -up to a version. - -All the nodes selective options are explained <>. +You can use this API to clear the archived repositories meterings in the cluster. [[clear-repositories-metering-archive-api-path-params]] ==== {api-path-parms-title} @@ -28,12 +24,12 @@ All the nodes selective options are explained <>. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] ``:: - (long) Specify the max version to be cleared from the archive. - -[[clear-repositories-metering-archive-api-query-params]] -==== {api-query-parms-title} + (long) Specifies the maximum <> to be cleared from the archive. +All the nodes selective options are explained <>. [role="child_attributes"] [[clear-repositories-metering-archive-api-response-body]] ==== {api-response-body-title} -It returns all the deleted repositories metrics. +Returns the deleted archived repositories meterings. + +include::{es-repo-dir}/repositories-metering-api/apis/repositories-meterings-body.asciidoc[tag=repositories-metering-body] diff --git a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc new file mode 100644 index 0000000000000..ff10a7fa0896d --- /dev/null +++ b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc @@ -0,0 +1,35 @@ +[role="xpack"] +[testenv="basic"] +[[get-repositories-metering-api]] +=== Get repositories meterings +++++ +Get repositories meterings +++++ + +Returns cluster repositories meterings. + +[[get-repositories-metering-api-request]] +==== {api-request-title} + +`GET /_nodes//_repositories_metering` + +[[get-repositories-metering-api-desc]] +==== {api-description-title} + +You can use the cluster repositories metering API to retrieve repositories metrics in a cluster. + +This API exposes monotonic increasing counters and it's expected that clients would durably store +the information needed to compute aggregations over a period of time. Additionally, the information +exposed by this API is volatile, meaning that it won't be present after node restarts. + +[[get-repositories-metering-api-path-params]] +==== {api-path-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] + +All the nodes selective options are explained <>. + +[role="child_attributes"] +[[get-repositories-metering-api-response-body]] +==== {api-response-body-title} +include::{es-repo-dir}/repositories-metering-api/apis/repositories-meterings-body.asciidoc[tag=repositories-metering-body] diff --git a/docs/reference/repositories-metering/apis/get-repositories-metering.asciidoc b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc similarity index 75% rename from docs/reference/repositories-metering/apis/get-repositories-metering.asciidoc rename to docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc index 005bbdfec9b68..0a1ced08f0e92 100644 --- a/docs/reference/repositories-metering/apis/get-repositories-metering.asciidoc +++ b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc @@ -1,37 +1,7 @@ -[role="xpack"] -[testenv="basic"] -[[get-repositories-metering-api]] -=== Repositories metering API -++++ -Repositories metering API -++++ - -Returns cluster repositories metering. - -[[get-repositories-metering-api-request]] -==== {api-request-title} - -`GET /_nodes//_repositories_metering` - -[[get-repositories-metering-api-desc]] -==== {api-description-title} - -You can use the cluster repositories metering API to retrieve repositories metrics in a cluster. - -All the nodes selective options are explained <>. - -[[get-repositories-metering-api-path-params]] -==== {api-path-parms-title} - -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] - -[[get-repositories-metering-api-query-params]] -==== {api-query-parms-title} - -[role="child_attributes"] -[[get-repositories-metering-api-response-body]] -==== {api-response-body-title} - +tag::repositories-metering-body[] +``:: +(Optional, string) Comma-separated list of node IDs or names used to limit +returned information. `_nodes`:: (object) Contains statistics about the number of nodes selected by the request. @@ -102,7 +72,7 @@ since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. (boolean) A flag that tells whether or not this object has been archived. -`cluster_version`:: +`cluster_version`:: (long) The cluster state version when this object was created, this field can be used as a logical timestamp to delete all the archived metrics up @@ -117,7 +87,6 @@ grouped by request type. .Properties of `request_counts` for repository type `Azure` [%collapsible%open] ====== -Azure storage https://azure.microsoft.com/en-us/pricing/details/storage/blobs/[pricing]. `GetBlobProperties`:: (long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-properties[Get Blob Properties] requests. `GetBlob`:: @@ -128,34 +97,39 @@ Azure storage https://azure.microsoft.com/en-us/pricing/details/storage/blobs/[p (long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob[Put Blob] requests. `PutBlobBlock`:: (long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-block[Put Block] requests. + +Azure storage https://azure.microsoft.com/en-us/pricing/details/storage/blobs/[pricing]. ====== + .Properties of `request_counts` for repository type `GCP` [%collapsible%open] ====== -Google Cloud Platform storage https://cloud.google.com/storage/pricing[pricing]. `storage.objects.get`:: -(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/get[GET] requests. +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/get[get] requests. `storage.objects.list`:: -(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/list[LIST] requests. +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/list[list] requests. `storage.objects.resumable-insert`:: -(long) Number of https://cloud.google.com/storage/docs/performing-resumable-uploads[PUT] requests. +(long) Number of https://cloud.google.com/storage/docs/performing-resumable-uploads[resumable insert] requests. `storage.objects.insert`:: -(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/insert[POST] requests. +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/insert[insert] requests. + +Google Cloud storage https://cloud.google.com/storage/pricing[pricing]. ====== + .Properties of `request_counts` for repository type `S3` [%collapsible%open] ====== -Amazon Web Services Simple Storage Service https://aws.amazon.com/s3/pricing/[pricing]. `GetObject`:: -(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html[GET] requests. +(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html[GetObject] requests. `ListObjects`:: -(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html[LIST] requests. +(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html[ListObjects] requests. `PutObject`:: -(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html[PUT] requests. +(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html[PutObject] requests. `PutMultipartObject`:: (long) Number of https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html[MultiPart] requests. + +Amazon Web Services Simple Storage Service https://aws.amazon.com/s3/pricing/[pricing]. ====== ===== ==== +end::repositories-metering-body[] diff --git a/docs/reference/repositories-metering/repositories-metering.asciidoc b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc similarity index 64% rename from docs/reference/repositories-metering/repositories-metering.asciidoc rename to docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc index d3477d15a03c4..20a62270001f2 100644 --- a/docs/reference/repositories-metering/repositories-metering.asciidoc +++ b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc @@ -5,11 +5,12 @@ experimental[] -You can use the following APIs to retrieve repositories metrics. +You can use the following APIs to retrieve repositories meterings. + This is an API meant to be used by Elastic's commercial offerings. * <> * <> -include::get-repositories-metering.asciidoc[] -include::clear-repositories-metering-archive.asciidoc[] +include::apis/get-repositories-metering.asciidoc[] +include::apis/clear-repositories-metering-archive.asciidoc[] diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc index 99ae417cff87b..cfe21493308de 100644 --- a/docs/reference/rest-api/index.asciidoc +++ b/docs/reference/rest-api/index.asciidoc @@ -30,6 +30,7 @@ endif::[] * <> * <> * <> +* <> * <> * <> ifdef::permanently-unreleased-branch[] @@ -63,6 +64,7 @@ include::{es-repo-dir}/ml/anomaly-detection/apis/index.asciidoc[] include::{es-repo-dir}/ml/df-analytics/apis/index.asciidoc[] include::{es-repo-dir}/migration/migration.asciidoc[] include::{es-repo-dir}/indices/apis/reload-analyzers.asciidoc[] +include::{es-repo-dir}/repositories-metering-api/repositories-metering-apis.asciidoc[] include::{es-repo-dir}/rollup/rollup-api.asciidoc[] include::{es-repo-dir}/search.asciidoc[] ifdef::permanently-unreleased-branch[] From 449069a77d017fd1df3c90c2f06fe322706773d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Wed, 2 Sep 2020 12:24:31 +0200 Subject: [PATCH 13/19] Address review comments --- ...ear-repositories-metering-archive.asciidoc | 2 +- .../apis/get-repositories-metering.asciidoc | 6 +- .../apis/repositories-meterings-body.asciidoc | 81 ++++++++++++++----- .../repositories-metering-apis.asciidoc | 2 +- .../repositories/azure/AzureRepository.java | 8 +- ...eCloudStorageBlobStoreRepositoryTests.java | 10 +-- .../GoogleCloudStorageOperationsStats.java | 8 +- .../gcs/GoogleCloudStorageRepository.java | 8 +- .../repositories/s3/S3Repository.java | 8 +- .../repositories/RepositoryInfo.java | 11 +-- .../blobstore/MeteredBlobStoreRepository.java | 16 +--- .../RepositoriesServiceTests.java | 4 +- .../RepositoriesStatsArchiveTests.java | 2 +- .../azure/AzureRepositoriesMeteringIT.java | 5 +- .../gcs/GCSRepositoriesMeteringIT.java | 9 ++- .../metering/s3/S3RepositoriesMeteringIT.java | 5 +- ...ctRepositoriesMeteringAPIRestTestCase.java | 4 +- .../RepositoriesMeteringResponseTests.java | 2 +- 18 files changed, 125 insertions(+), 66 deletions(-) diff --git a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc index 7c9cc4553343b..6e80267c3869e 100644 --- a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc +++ b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc @@ -6,7 +6,7 @@ Clear repositories metering archive ++++ -Removes the archived repositories meterings present in the cluster. +Removes the archived repositories metering information present in the cluster. [[clear-repositories-metering-archive-api-request]] ==== {api-request-title} diff --git a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc index ff10a7fa0896d..3485a9a4b7bec 100644 --- a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc +++ b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc @@ -6,7 +6,7 @@ Get repositories meterings ++++ -Returns cluster repositories meterings. +Returns cluster repositories metering information. [[get-repositories-metering-api-request]] ==== {api-request-title} @@ -16,9 +16,9 @@ Returns cluster repositories meterings. [[get-repositories-metering-api-desc]] ==== {api-description-title} -You can use the cluster repositories metering API to retrieve repositories metrics in a cluster. +You can use the cluster repositories metering API to retrieve repositories metering information in a cluster. -This API exposes monotonic increasing counters and it's expected that clients would durably store +This API exposes monotonically non-decreasing counters and it's expected that clients would durably store the information needed to compute aggregations over a period of time. Additionally, the information exposed by this API is volatile, meaning that it won't be present after node restarts. diff --git a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc index 0a1ced08f0e92..86f5a5783e038 100644 --- a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc +++ b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc @@ -50,10 +50,44 @@ Repository name. Repository type. `repository_location`:: -(string) -Represents an unique location within the repository, -i.e. a `bucket` + `base_path`. +(object) +Represents an unique location within the repository. ++ +.Properties of `repository_location` for repository type `Azure` +[%collapsible%open] +====== +`base_path`:: +(string) +The path within the container where the repository stores data. + +`container`:: +(string) +Container name. +====== ++ +.Properties of `repository_location` for repository type `GCP` +[%collapsible%open] +====== +`base_path`:: +(string) +The path within the bucket where the repository stores data. + +`bucket`:: +(string) +Bucket name. +====== ++ +.Properties of `repository_location` for repository type `S3` +[%collapsible%open] +====== +`base_path`:: +(string) +The path within the bucket where the repository stores data. +`bucket`:: +(string) +Bucket name. +====== `repository_ephemeral_id`:: (string) An identifier that changes every time the repository is updated. @@ -71,13 +105,19 @@ since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. `archived`:: (boolean) A flag that tells whether or not this object has been archived. +When a repository is closed or updated the repository metering information +is archived and kept for a certain period of time. This allows retrieving +the repository metering information of previous repository versions. -`cluster_version`:: +`archive_version`:: (long) -The cluster state version when this object was created, this field +The cluster state version when this object was archived, this field can be used as a logical timestamp to delete all the archived metrics up -to a observed cluster version. This field is only present for archived -objects. +to an observed version. This field is only present for archived +repository metering information objects. The main purpose of this +field is to avoid possible race conditions during repository metering +information deletions, i.e. deleting archived repositories metering +information that we haven't observed yet. `request_counts`:: (object) @@ -95,8 +135,8 @@ grouped by request type. (long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/list-blobs[List Blobs] requests. `PutBlob`:: (long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob[Put Blob] requests. -`PutBlobBlock`:: -(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-block[Put Block] requests. +`PutBlock`:: +(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-block[Put Block] and https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list[Put Block List] requests. Azure storage https://azure.microsoft.com/en-us/pricing/details/storage/blobs/[pricing]. ====== @@ -104,14 +144,16 @@ Azure storage https://azure.microsoft.com/en-us/pricing/details/storage/blobs/[p .Properties of `request_counts` for repository type `GCP` [%collapsible%open] ====== -`storage.objects.get`:: -(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/get[get] requests. -`storage.objects.list`:: -(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/list[list] requests. -`storage.objects.resumable-insert`:: -(long) Number of https://cloud.google.com/storage/docs/performing-resumable-uploads[resumable insert] requests. -`storage.objects.insert`:: -(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/insert[insert] requests. +`GetObject`:: +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/get[get object] requests. +`ListObjects`:: +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/list[list objects] requests. +`ResumableInsertObject`:: +(long) Number of https://cloud.google.com/storage/docs/performing-resumable-uploads[resumable insert object] requests, +even though resumable uploads can perform multiple http requests they are considered as a single operation for billing +purposes. +`InsertObject`:: +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/insert[insert object] requests. Google Cloud storage https://cloud.google.com/storage/pricing[pricing]. ====== @@ -126,7 +168,10 @@ Google Cloud storage https://cloud.google.com/storage/pricing[pricing]. `PutObject`:: (long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html[PutObject] requests. `PutMultipartObject`:: -(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html[MultiPart] requests. +(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html[MultiPart] requests, +including https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html[CreateMultiPartUpload], +https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html[UploadPart] and https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html[CompleteMultipartUpload] +requests. Amazon Web Services Simple Storage Service https://aws.amazon.com/s3/pricing/[pricing]. ====== diff --git a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc index 20a62270001f2..cb406bbcb1a05 100644 --- a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc +++ b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc @@ -7,7 +7,7 @@ experimental[] You can use the following APIs to retrieve repositories meterings. -This is an API meant to be used by Elastic's commercial offerings. +This is an API used by Elastic's commercial offerings. * <> * <> diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 7b51c08629f35..cd3e534ec62ac 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -36,6 +36,7 @@ import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import java.util.Locale; +import java.util.Map; import java.util.function.Function; import static org.elasticsearch.repositories.azure.AzureStorageService.MAX_CHUNK_SIZE; @@ -87,7 +88,7 @@ public AzureRepository( clusterService, recoverySettings, buildBasePath(metadata), - Repository.CONTAINER_SETTING.get(metadata.settings())); + buildLocation(metadata)); this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings()); this.storageService = storageService; @@ -115,6 +116,11 @@ private static BlobPath buildBasePath(RepositoryMetadata metadata) { } } + private static Map buildLocation(RepositoryMetadata metadata) { + return Map.of("base_path", buildBasePath(metadata).buildAsString(), + "container", Repository.CONTAINER_SETTING.get(metadata.settings())); + } + @Override protected BlobStore getBlobStore() { return super.getBlobStore(); diff --git a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index f65be55f21553..47b179ba47299 100644 --- a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -310,15 +310,15 @@ private static class GoogleCloudStorageStatsCollectorHttpHandler extends HttpSta @Override public void maybeTrack(final String request, Headers requestHeaders) { if (Regex.simpleMatch("GET /storage/v1/b/*/o/*", request)) { - trackRequest("storage.objects.get"); + trackRequest("GetObject"); } else if (Regex.simpleMatch("GET /storage/v1/b/*/o*", request)) { - trackRequest("storage.objects.list"); + trackRequest("ListObjects"); } else if (Regex.simpleMatch("GET /download/storage/v1/b/*", request)) { - trackRequest("storage.objects.get"); + trackRequest("GetObject"); } else if (Regex.simpleMatch("PUT /upload/storage/v1/b/*", request) && isLastPart(requestHeaders)) { - trackRequest("storage.objects.resumable-insert"); + trackRequest("ResumableInsertObject"); } else if (Regex.simpleMatch("POST /upload/storage/v1/b/*uploadType=multipart*", request)) { - trackRequest("storage.objects.insert"); + trackRequest("InsertObject"); } } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java index d0c83f50bceb4..dc87a77e911a9 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java @@ -58,10 +58,10 @@ String getTrackedBucket() { Map toMap() { final Map results = new HashMap<>(); - results.put("storage.objects.get", getCount.get()); - results.put("storage.objects.list", listCount.get()); - results.put("storage.objects.resumable-insert", putCount.get()); - results.put("storage.objects.insert", postCount.get()); + results.put("GetObject", getCount.get()); + results.put("ListObjects", listCount.get()); + results.put("ResumableInsertObject", putCount.get()); + results.put("InsertObject", postCount.get()); return results; } } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index 5565a1d86a765..c6ade380e3a9c 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -33,6 +33,7 @@ import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; +import java.util.Map; import java.util.function.Function; import static org.elasticsearch.common.settings.Setting.Property; @@ -72,7 +73,7 @@ class GoogleCloudStorageRepository extends MeteredBlobStoreRepository { final GoogleCloudStorageService storageService, final ClusterService clusterService, final RecoverySettings recoverySettings) { - super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildBasePath(metadata), getSetting(BUCKET, metadata)); + super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildBasePath(metadata), buildLocation(metadata)); this.storageService = storageService; this.chunkSize = getSetting(CHUNK_SIZE, metadata); @@ -95,6 +96,11 @@ private static BlobPath buildBasePath(RepositoryMetadata metadata) { } } + private static Map buildLocation(RepositoryMetadata metadata) { + return Map.of("base_path", buildBasePath(metadata).buildAsString(), + "bucket", getSetting(BUCKET, metadata)); + } + @Override protected GoogleCloudStorageBlobStore createBlobStore() { return new GoogleCloudStorageBlobStore(bucket, clientName, metadata.name(), storageService, bufferSize); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index e80236e4501e8..7717a9e0f5fbe 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -49,6 +49,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.util.Collection; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -199,7 +200,7 @@ class S3Repository extends MeteredBlobStoreRepository { clusterService, recoverySettings, buildBasePath(metadata), - BUCKET_SETTING.get(metadata.settings())); + buildLocation(metadata)); this.service = service; // Parse and validate the user's S3 Storage Class setting @@ -234,6 +235,11 @@ class S3Repository extends MeteredBlobStoreRepository { storageClass); } + private static Map buildLocation(RepositoryMetadata metadata) { + return Map.of("base_path", buildBasePath(metadata).buildAsString(), + "bucket", BUCKET_SETTING.get(metadata.settings())); + } + /** * Holds a reference to delayed repository operation {@link Scheduler.Cancellable} so it can be cancelled should the repository be * closed concurrently. diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java index 2361767dcad1b..8d2612ba70b04 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryInfo.java @@ -28,13 +28,14 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.Objects; public final class RepositoryInfo implements Writeable, ToXContentFragment { public final String ephemeralId; public final String name; public final String type; - public final String location; + public final Map location; public final long startedAt; @Nullable public final Long stoppedAt; @@ -42,7 +43,7 @@ public final class RepositoryInfo implements Writeable, ToXContentFragment { public RepositoryInfo(String ephemeralId, String name, String type, - String location, + Map location, long startedAt) { this(ephemeralId, name, type, location, startedAt, null); } @@ -50,7 +51,7 @@ public RepositoryInfo(String ephemeralId, public RepositoryInfo(String ephemeralId, String name, String type, - String location, + Map location, long startedAt, @Nullable Long stoppedAt) { this.ephemeralId = ephemeralId; @@ -68,7 +69,7 @@ public RepositoryInfo(StreamInput in) throws IOException { this.ephemeralId = in.readString(); this.name = in.readString(); this.type = in.readString(); - this.location = in.readString(); + this.location = in.readMap(StreamInput::readString, StreamInput::readString); this.startedAt = in.readLong(); this.stoppedAt = in.readOptionalLong(); } @@ -88,7 +89,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(ephemeralId); out.writeString(name); out.writeString(type); - out.writeString(location); + out.writeMap(location, StreamOutput::writeString, StreamOutput::writeString); out.writeLong(startedAt); out.writeOptionalLong(stoppedAt); } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java index bf19ad90c745c..b460f373d86b9 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java @@ -29,6 +29,8 @@ import org.elasticsearch.repositories.RepositoryStatsSnapshot; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Map; + public abstract class MeteredBlobStoreRepository extends BlobStoreRepository { private final RepositoryInfo repositoryInfo; @@ -37,13 +39,13 @@ public MeteredBlobStoreRepository(RepositoryMetadata metadata, ClusterService clusterService, RecoverySettings recoverySettings, BlobPath basePath, - String bucket) { + Map location) { super(metadata, namedXContentRegistry, clusterService, recoverySettings, basePath); ThreadPool threadPool = clusterService.getClusterApplierService().threadPool(); this.repositoryInfo = new RepositoryInfo(UUIDs.randomBase64UUID(), metadata.name(), metadata.type(), - getLocation(basePath, bucket), + location, threadPool.absoluteTimeInMillis()); } @@ -55,14 +57,4 @@ public RepositoryStatsSnapshot statsSnapshotForArchival(long clusterVersion) { RepositoryInfo stoppedRepoInfo = repositoryInfo.stopped(threadPool.absoluteTimeInMillis()); return new RepositoryStatsSnapshot(stoppedRepoInfo, stats(), clusterVersion, true); } - - private static String getLocation(BlobPath basePath, String bucket) { - BlobPath location = BlobPath.cleanPath(); - - location = location.add(bucket); - for (String path : basePath) { - location = location.add(path); - } - return location.buildAsString(); - } } diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 5e022f7605ae3..2c74ad03db626 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -328,7 +328,7 @@ private MeteredRepositoryTypeA(RepositoryMetadata metadata, ClusterService clust clusterService, mock(RecoverySettings.class), BlobPath.cleanPath(), - "bucket-a"); + Map.of("bucket", "bucket-a")); } @Override @@ -352,7 +352,7 @@ private MeteredRepositoryTypeB(RepositoryMetadata metadata, ClusterService clust clusterService, mock(RecoverySettings.class), BlobPath.cleanPath(), - "bucket-b"); + Map.of("bucket", "bucket-b")); } @Override diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java index a17b99f2b8b7f..2c9294d853d81 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesStatsArchiveTests.java @@ -108,7 +108,7 @@ private RepositoryStatsSnapshot createRepositoryStats(RepositoryStats repository RepositoryInfo repositoryInfo = new RepositoryInfo(UUIDs.randomBase64UUID(), randomAlphaOfLength(10), randomAlphaOfLength(10), - randomAlphaOfLength(10), + Map.of("bucket", randomAlphaOfLength(10)), System.currentTimeMillis(), null); return new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, clusterVersion, true); diff --git a/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java index 1e95dac5781dc..f5ad262f8098d 100644 --- a/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.repositories.metering.AbstractRepositoriesMeteringAPIRestTestCase; import java.util.List; +import java.util.Map; public class AzureRepositoriesMeteringIT extends AbstractRepositoriesMeteringAPIRestTestCase { @@ -18,8 +19,8 @@ protected String repositoryType() { } @Override - protected String repositoryLocation() { - return getProperty("test.azure.container") + "/" + getProperty("test.azure.base_path") + "/"; + protected Map repositoryLocation() { + return Map.of("container", getProperty("test.azure.container"), "base_path", getProperty("test.azure.base_path") + "/"); } @Override diff --git a/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java index 63a031d921e8a..d00cb8ab6c4af 100644 --- a/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.repositories.metering.AbstractRepositoriesMeteringAPIRestTestCase; import java.util.List; +import java.util.Map; public class GCSRepositoriesMeteringIT extends AbstractRepositoriesMeteringAPIRestTestCase { @@ -18,8 +19,8 @@ protected String repositoryType() { } @Override - protected String repositoryLocation() { - return getProperty("test.gcs.bucket") + "/" + getProperty("test.gcs.base_path") + "/"; + protected Map repositoryLocation() { + return Map.of("bucket", getProperty("test.gcs.bucket"), "base_path", getProperty("test.gcs.base_path") + "/"); } @Override @@ -37,11 +38,11 @@ protected Settings updatedRepositorySettings() { @Override protected List readCounterKeys() { - return List.of("storage.objects.get", "storage.objects.list"); + return List.of("GetObject", "ListObjects"); } @Override protected List writeCounterKeys() { - return List.of("storage.objects.insert"); + return List.of("InsertObject"); } } diff --git a/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java index a5d7cb1a5c8d9..a18a6e7889b73 100644 --- a/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.repositories.metering.AbstractRepositoriesMeteringAPIRestTestCase; import java.util.List; +import java.util.Map; public class S3RepositoriesMeteringIT extends AbstractRepositoriesMeteringAPIRestTestCase { @@ -18,8 +19,8 @@ protected String repositoryType() { } @Override - protected String repositoryLocation() { - return getProperty("test.s3.bucket") + "/" + getProperty("test.s3.base_path") + "/"; + protected Map repositoryLocation() { + return Map.of("bucket", getProperty("test.s3.bucket"), "base_path", getProperty("test.s3.base_path") + "/"); } @Override diff --git a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/AbstractRepositoriesMeteringAPIRestTestCase.java b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/AbstractRepositoriesMeteringAPIRestTestCase.java index bfad5d87f00db..a716d22f15953 100644 --- a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/AbstractRepositoriesMeteringAPIRestTestCase.java +++ b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/AbstractRepositoriesMeteringAPIRestTestCase.java @@ -40,7 +40,7 @@ public abstract class AbstractRepositoriesMeteringAPIRestTestCase extends ESRestTestCase { protected abstract String repositoryType(); - protected abstract String repositoryLocation(); + protected abstract Map repositoryLocation(); protected abstract Settings repositorySettings(); @@ -339,7 +339,7 @@ private RepositoryInfo parseRepositoryInfo(Map nodeStatSnapshot) String id = extractValue(nodeStatSnapshot, "repository_ephemeral_id"); String name = extractValue(nodeStatSnapshot, "repository_name"); String type = extractValue(nodeStatSnapshot, "repository_type"); - String location = extractValue(nodeStatSnapshot, "repository_location"); + Map location = extractValue(nodeStatSnapshot, "repository_location"); Long startedAt = extractValue(nodeStatSnapshot, "repository_started_at"); Long stoppedAt = extractValue(nodeStatSnapshot, "repository_stopped_at"); return new RepositoryInfo(id, name, type, location, startedAt, stoppedAt); diff --git a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java index 354ba5dfab4ed..e68aa5bf5a283 100644 --- a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java +++ b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java @@ -68,7 +68,7 @@ private RepositoriesMeteringResponse createResponse() { String repoId = randomAlphaOfLength(10); String repoName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); String repoType = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); - String repoLocation = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + Map repoLocation = Map.of("bucket", randomAlphaOfLength(10).toLowerCase(Locale.ROOT)); long startedAt = System.currentTimeMillis() - 1; Long stoppedAt = randomBoolean() ? System.currentTimeMillis() : null; RepositoryInfo repositoryInfo = new RepositoryInfo(repoId, repoName, repoType, repoLocation, startedAt, stoppedAt); From bc94df15bc86867c11069e56872ee95fe05dcde3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Thu, 3 Sep 2020 13:21:29 +0200 Subject: [PATCH 14/19] Address review comments --- ...ear-repositories-metering-archive.asciidoc | 2 +- .../apis/repositories-meterings-body.asciidoc | 14 +++++------ .../azure/AzureBlobStoreRepositoryTests.java | 15 +++++++---- .../repositories/azure/AzureBlobStore.java | 25 +++++++++++-------- ...eCloudStorageBlobStoreRepositoryTests.java | 9 ++++--- .../GoogleCloudStorageOperationsStats.java | 3 +-- 6 files changed, 38 insertions(+), 30 deletions(-) diff --git a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc index 6e80267c3869e..8254f16ac2867 100644 --- a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc +++ b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc @@ -24,7 +24,7 @@ You can use this API to clear the archived repositories meterings in the cluster include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] ``:: - (long) Specifies the maximum <> to be cleared from the archive. + (long) Specifies the maximum <> to be cleared from the archive. All the nodes selective options are explained <>. [role="child_attributes"] diff --git a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc index 86f5a5783e038..741db5ad6161b 100644 --- a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc +++ b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc @@ -107,7 +107,7 @@ since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. A flag that tells whether or not this object has been archived. When a repository is closed or updated the repository metering information is archived and kept for a certain period of time. This allows retrieving -the repository metering information of previous repository versions. +the repository metering information of previous repository instantiations. `archive_version`:: (long) @@ -136,7 +136,9 @@ grouped by request type. `PutBlob`:: (long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob[Put Blob] requests. `PutBlock`:: -(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-block[Put Block] and https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list[Put Block List] requests. +(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-block[Put Block]. +`PutBlockList`:: +(long) Number of https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list[Put Block List] requests. Azure storage https://azure.microsoft.com/en-us/pricing/details/storage/blobs/[pricing]. ====== @@ -148,10 +150,6 @@ Azure storage https://azure.microsoft.com/en-us/pricing/details/storage/blobs/[p (long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/get[get object] requests. `ListObjects`:: (long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/list[list objects] requests. -`ResumableInsertObject`:: -(long) Number of https://cloud.google.com/storage/docs/performing-resumable-uploads[resumable insert object] requests, -even though resumable uploads can perform multiple http requests they are considered as a single operation for billing -purposes. `InsertObject`:: (long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/insert[insert object] requests. @@ -168,8 +166,8 @@ Google Cloud storage https://cloud.google.com/storage/pricing[pricing]. `PutObject`:: (long) Number of https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html[PutObject] requests. `PutMultipartObject`:: -(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html[MultiPart] requests, -including https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html[CreateMultiPartUpload], +(long) Number of https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html[Multipart] requests, +including https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html[CreateMultipartUpload], https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html[UploadPart] and https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html[CompleteMultipartUpload] requests. diff --git a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java index 88e5f99d4a605..aee308597c756 100644 --- a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -186,18 +186,23 @@ protected void maybeTrack(String request, Headers headers) { trackRequest("GetBlobProperties"); } else if (listPattern.test(request)) { trackRequest("ListBlobs"); - } else if (isBlockUpload(request)) { + } else if (isPutBlock(request)) { trackRequest("PutBlock"); + } else if (isPutBlockList(request)) { + trackRequest("PutBlockList"); } else if (Regex.simpleMatch("PUT /*/*", request)) { trackRequest("PutBlob"); } } - // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block - private boolean isBlockUpload(String request) { - return Regex.simpleMatch("PUT /*/*?*comp=blocklist*", request) - || (Regex.simpleMatch("PUT /*/*?*comp=block*", request) && request.contains("blockid=")); + private boolean isPutBlock(String request) { + return Regex.simpleMatch("PUT /*/*?*comp=block*", request) && request.contains("blockid="); + } + + // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list + private boolean isPutBlockList(String request) { + return Regex.simpleMatch("PUT /*/*?*comp=blocklist*", request); } } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java index 6521fb2d6eae5..aa8df87b99612 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java @@ -118,19 +118,19 @@ public AzureBlobStore(RepositoryMetadata metadata, AzureStorageService service, this.uploadMetricsCollector = (httpURLConnection -> { assert httpURLConnection.getRequestMethod().equals("PUT"); String queryParams = httpURLConnection.getURL().getQuery(); - if (queryParams != null && isBlockUpload(queryParams)) { - stats.putBlockOperations.incrementAndGet(); - } else { + if (queryParams == null) { stats.putOperations.incrementAndGet(); + return; } - }); - } - private boolean isBlockUpload(String queryParams) { - // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block - // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list - return (queryParams.contains("comp=block") && queryParams.contains("blockid=")) - || queryParams.contains("comp=blocklist"); + // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block + // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list + if (queryParams.contains("comp=block") && queryParams.contains("blockid=")) { + stats.putBlockOperations.incrementAndGet(); + } else if (queryParams.contains("comp=blocklist")) { + stats.putBlockListOperations.incrementAndGet(); + } + }); } @Override @@ -383,12 +383,15 @@ private static class Stats { private final AtomicLong putBlockOperations = new AtomicLong(); + private final AtomicLong putBlockListOperations = new AtomicLong(); + private Map toMap() { return Map.of("GetBlob", getOperations.get(), "ListBlobs", listOperations.get(), "GetBlobProperties", headOperations.get(), "PutBlob", putOperations.get(), - "PutBlock", putBlockOperations.get()); + "PutBlock", putBlockOperations.get(), + "PutBlockList", putBlockListOperations.get()); } } } diff --git a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 47b179ba47299..34eda606d39ad 100644 --- a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -312,11 +312,14 @@ public void maybeTrack(final String request, Headers requestHeaders) { if (Regex.simpleMatch("GET /storage/v1/b/*/o/*", request)) { trackRequest("GetObject"); } else if (Regex.simpleMatch("GET /storage/v1/b/*/o*", request)) { - trackRequest("ListObjects"); + trackRequest("ListObjGects"); } else if (Regex.simpleMatch("GET /download/storage/v1/b/*", request)) { trackRequest("GetObject"); - } else if (Regex.simpleMatch("PUT /upload/storage/v1/b/*", request) && isLastPart(requestHeaders)) { - trackRequest("ResumableInsertObject"); + } else if (Regex.simpleMatch("PUT /upload/storage/v1/b/*uploadType=resumable*", request) && isLastPart(requestHeaders)) { + // Resumable uploads are billed as a single operation, that's the reason we're tracking + // the request only when it's the last part. + // See https://cloud.google.com/storage/docs/resumable-uploads#introduction + trackRequest("InsertObject"); } else if (Regex.simpleMatch("POST /upload/storage/v1/b/*uploadType=multipart*", request)) { trackRequest("InsertObject"); } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java index dc87a77e911a9..2264f71c29f4a 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java @@ -60,8 +60,7 @@ Map toMap() { final Map results = new HashMap<>(); results.put("GetObject", getCount.get()); results.put("ListObjects", listCount.get()); - results.put("ResumableInsertObject", putCount.get()); - results.put("InsertObject", postCount.get()); + results.put("InsertObject", postCount.get() + putCount.get()); return results; } } From e53322b42232e58b63ce6c4837e39e83cf699a54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Thu, 3 Sep 2020 14:35:28 +0200 Subject: [PATCH 15/19] Use raw base_path for Repository metering API --- .../org/elasticsearch/repositories/azure/AzureRepository.java | 2 +- .../repositories/gcs/GoogleCloudStorageRepository.java | 2 +- .../java/org/elasticsearch/repositories/s3/S3Repository.java | 2 +- .../metering/azure/AzureRepositoriesMeteringIT.java | 2 +- .../repositories/metering/gcs/GCSRepositoriesMeteringIT.java | 2 +- .../repositories/metering/s3/S3RepositoriesMeteringIT.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index cd3e534ec62ac..50b2d04ad2420 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -117,7 +117,7 @@ private static BlobPath buildBasePath(RepositoryMetadata metadata) { } private static Map buildLocation(RepositoryMetadata metadata) { - return Map.of("base_path", buildBasePath(metadata).buildAsString(), + return Map.of("base_path", Repository.BASE_PATH_SETTING.get(metadata.settings()), "container", Repository.CONTAINER_SETTING.get(metadata.settings())); } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index c6ade380e3a9c..6cea3446a26a6 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -97,7 +97,7 @@ private static BlobPath buildBasePath(RepositoryMetadata metadata) { } private static Map buildLocation(RepositoryMetadata metadata) { - return Map.of("base_path", buildBasePath(metadata).buildAsString(), + return Map.of("base_path", BASE_PATH.get(metadata.settings()), "bucket", getSetting(BUCKET, metadata)); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 7717a9e0f5fbe..802957e66818e 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -236,7 +236,7 @@ class S3Repository extends MeteredBlobStoreRepository { } private static Map buildLocation(RepositoryMetadata metadata) { - return Map.of("base_path", buildBasePath(metadata).buildAsString(), + return Map.of("base_path", BASE_PATH_SETTING.get(metadata.settings()), "bucket", BUCKET_SETTING.get(metadata.settings())); } diff --git a/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java index f5ad262f8098d..7be78bdabcd1c 100644 --- a/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/azure/src/test/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java @@ -20,7 +20,7 @@ protected String repositoryType() { @Override protected Map repositoryLocation() { - return Map.of("container", getProperty("test.azure.container"), "base_path", getProperty("test.azure.base_path") + "/"); + return Map.of("container", getProperty("test.azure.container"), "base_path", getProperty("test.azure.base_path")); } @Override diff --git a/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java index d00cb8ab6c4af..f843410293b3f 100644 --- a/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/gcs/src/test/java/org/elasticsearch/xpack/repositories/metering/gcs/GCSRepositoriesMeteringIT.java @@ -20,7 +20,7 @@ protected String repositoryType() { @Override protected Map repositoryLocation() { - return Map.of("bucket", getProperty("test.gcs.bucket"), "base_path", getProperty("test.gcs.base_path") + "/"); + return Map.of("bucket", getProperty("test.gcs.bucket"), "base_path", getProperty("test.gcs.base_path")); } @Override diff --git a/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java index a18a6e7889b73..777fed93286ef 100644 --- a/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java +++ b/x-pack/plugin/repositories-metering-api/qa/s3/src/test/java/org/elasticsearch/xpack/repositories/metering/s3/S3RepositoriesMeteringIT.java @@ -20,7 +20,7 @@ protected String repositoryType() { @Override protected Map repositoryLocation() { - return Map.of("bucket", getProperty("test.s3.bucket"), "base_path", getProperty("test.s3.base_path") + "/"); + return Map.of("bucket", getProperty("test.s3.bucket"), "base_path", getProperty("test.s3.base_path")); } @Override From 1b798288e3947f280e6acb8d352fa4c8cf2974c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Fri, 4 Sep 2020 11:18:19 +0200 Subject: [PATCH 16/19] minor corrections --- .../apis/repositories-meterings-body.asciidoc | 11 ++++------- .../repositories-metering-apis.asciidoc | 2 +- .../repositories/RepositoryStatsSnapshot.java | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc index 741db5ad6161b..d840d476ead55 100644 --- a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc +++ b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc @@ -1,7 +1,4 @@ tag::repositories-metering-body[] -``:: -(Optional, string) Comma-separated list of node IDs or names used to limit -returned information. `_nodes`:: (object) Contains statistics about the number of nodes selected by the request. @@ -29,14 +26,14 @@ Name of the cluster. Based on the <> setting. `nodes`:: (object) -Contains repositories metrics for the nodes selected by the request. +Contains repositories metering information for the nodes selected by the request. + .Properties of `nodes` [%collapsible%open] ==== ``:: (array) -An array of repository metrics for the node. +An array of repository metering information for the node. + .Properties of objects in `node_id` [%collapsible%open] @@ -98,7 +95,7 @@ Time the repository was created or updated. Recorded in milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. `repository_stopped_at`:: -(long) +(Optional, long) Time the repository was deleted or updated. Recorded in milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. @@ -110,7 +107,7 @@ is archived and kept for a certain period of time. This allows retrieving the repository metering information of previous repository instantiations. `archive_version`:: -(long) +(Optional, long) The cluster state version when this object was archived, this field can be used as a logical timestamp to delete all the archived metrics up to an observed version. This field is only present for archived diff --git a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc index cb406bbcb1a05..58f84b035ddcd 100644 --- a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc +++ b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc @@ -5,7 +5,7 @@ experimental[] -You can use the following APIs to retrieve repositories meterings. +You can use the following APIs to retrieve repositories metering information. This is an API used by Elastic's commercial offerings. diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java index df42d76d80de6..55a13e5fde0f7 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryStatsSnapshot.java @@ -23,13 +23,13 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.util.Objects; -public final class RepositoryStatsSnapshot implements Writeable, ToXContent { +public final class RepositoryStatsSnapshot implements Writeable, ToXContentObject { public static final long UNKNOWN_CLUSTER_VERSION = -1; private final RepositoryInfo repositoryInfo; private final RepositoryStats repositoryStats; From f9fa36984ab63bbdc57785dd582d13953dc80f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Fri, 4 Sep 2020 11:36:21 +0200 Subject: [PATCH 17/19] Minor correction --- .../elasticsearch/repositories/RepositoriesStatsArchive.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java index 68ac085c89e71..d0af94b4155fd 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStatsArchive.java @@ -95,7 +95,7 @@ private void evict() { ArchiveEntry entry; while ((entry = archive.peek()) != null && entry.ageInMillis(relativeTimeSupplier) >= retentionPeriod.getMillis()) { ArchiveEntry removedEntry = archive.poll(); - logger.debug("Evicting repository stats [{}]", removedEntry); + logger.debug("Evicting repository stats [{}]", removedEntry.repositoryStatsSnapshot); } } From c3a1f3ac616e0be6861f0c973cb51b495d29bf71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Fri, 4 Sep 2020 15:52:12 +0200 Subject: [PATCH 18/19] Fix testRequestStats. Mention uploads tracked on GCS InsertObject. --- .../apis/repositories-meterings-body.asciidoc | 5 +- .../azure/AzureBlobStoreRepositoryTests.java | 6 --- ...eCloudStorageBlobStoreRepositoryTests.java | 8 +--- .../s3/S3BlobStoreRepositoryTests.java | 5 -- ...ESMockAPIBasedRepositoryIntegTestCase.java | 46 ++++++++----------- 5 files changed, 24 insertions(+), 46 deletions(-) diff --git a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc index d840d476ead55..9547718c1d495 100644 --- a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc +++ b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc @@ -148,7 +148,10 @@ Azure storage https://azure.microsoft.com/en-us/pricing/details/storage/blobs/[p `ListObjects`:: (long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/list[list objects] requests. `InsertObject`:: -(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/insert[insert object] requests. +(long) Number of https://cloud.google.com/storage/docs/json_api/v1/objects/insert[insert object] requests, +including https://cloud.google.com/storage/docs/uploading-objects[simple], https://cloud.google.com/storage/docs/json_api/v1/how-tos/multipart-upload[multipart] and +https://cloud.google.com/storage/docs/resumable-uploads[resumable] uploads. Resumable uploads can perform multiple http requests to +insert a single object but they are considered as a single request since they are https://cloud.google.com/storage/docs/resumable-uploads#introduction[billed] as an individual operation. Google Cloud storage https://cloud.google.com/storage/pricing[pricing]. ====== diff --git a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java index aee308597c756..3e305c1cb1baf 100644 --- a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -40,7 +40,6 @@ import java.util.Base64; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -77,11 +76,6 @@ protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) { return new AzureErroneousHttpHandler(delegate, randomIntBetween(2, 3)); } - @Override - protected List requestTypesTracked() { - return List.of("GET", "LIST", "HEAD", "PUT", "PUT_BLOCK"); - } - @Override protected Settings nodeSettings(int nodeOrdinal) { final String key = Base64.getEncoder().encodeToString(randomAlphaOfLength(10).getBytes(StandardCharsets.UTF_8)); diff --git a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 34eda606d39ad..5d193706c8d8a 100644 --- a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -59,7 +59,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -105,11 +104,6 @@ protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) { return new GoogleErroneousHttpHandler(delegate, randomIntBetween(2, 3)); } - @Override - protected List requestTypesTracked() { - return List.of("GET", "LIST", "POST", "PUT"); - } - @Override protected Settings nodeSettings(int nodeOrdinal) { final Settings.Builder settings = Settings.builder(); @@ -312,7 +306,7 @@ public void maybeTrack(final String request, Headers requestHeaders) { if (Regex.simpleMatch("GET /storage/v1/b/*/o/*", request)) { trackRequest("GetObject"); } else if (Regex.simpleMatch("GET /storage/v1/b/*/o*", request)) { - trackRequest("ListObjGects"); + trackRequest("ListObjects"); } else if (Regex.simpleMatch("GET /download/storage/v1/b/*", request)) { trackRequest("GetObject"); } else if (Regex.simpleMatch("PUT /upload/storage/v1/b/*uploadType=resumable*", request) && isLastPart(requestHeaders)) { diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 2a8b7dafd70af..d96cf762e0d60 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -119,11 +119,6 @@ protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) { return new S3StatsCollectorHttpHandler(new S3ErroneousHttpHandler(delegate, randomIntBetween(2, 3))); } - @Override - protected List requestTypesTracked() { - return List.of("GET", "LIST", "PUT", "POST"); - } - @Override protected Settings nodeSettings(int nodeOrdinal) { final MockSecureSettings secureSettings = new MockSecureSettings(); diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java index 03a69db208530..e174d94d3716a 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java @@ -49,6 +49,7 @@ import java.io.InputStream; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -135,8 +136,6 @@ public void tearDownHttpServer() { protected abstract HttpHandler createErroneousHttpHandler(HttpHandler delegate); - protected abstract List requestTypesTracked(); - /** * Test the snapshot and restore of an index which has large segments files. */ @@ -217,32 +216,25 @@ public void testRequestStats() throws Exception { Map sdkRequestCounts = repositoryStats.requestCounts; - for (String requestType : requestTypesTracked()) { - assertSDKCallsMatchMockCalls(sdkRequestCounts, requestType); - } - } + final Map mockCalls = getMockRequestCounts(); - private void assertSDKCallsMatchMockCalls(Map sdkRequestCount, String requestTye) { - final long sdkCalls = sdkRequestCount.getOrDefault(requestTye, 0L); - final long mockCalls = handlers.values().stream() - .mapToLong(h -> { - while (h instanceof DelegatingHttpHandler) { - if (h instanceof HttpStatsCollectorHandler) { - return ((HttpStatsCollectorHandler) h).getCount(requestTye); - } - h = ((DelegatingHttpHandler) h).getDelegate(); - } + String assertionErrorMsg = String.format("SDK sent [%s] calls and handler measured [%s] calls", + sdkRequestCounts, + mockCalls); - return 0L; - }).sum(); - - String assertionErrorMsg = String.format("SDK sent %d [%s] calls and handler measured %d [%s] calls", - sdkCalls, - requestTye, - mockCalls, - requestTye); + assertEquals(assertionErrorMsg, mockCalls, sdkRequestCounts); + } - assertEquals(assertionErrorMsg, mockCalls, sdkCalls); + private Map getMockRequestCounts() { + for (HttpHandler h : handlers.values()) { + while (h instanceof DelegatingHttpHandler) { + if (h instanceof HttpStatsCollectorHandler) { + return ((HttpStatsCollectorHandler) h).getOperationsCount(); + } + h = ((DelegatingHttpHandler) h).getDelegate(); + } + } + return Collections.emptyMap(); } protected static String httpServerUrl() { @@ -352,8 +344,8 @@ public HttpHandler getDelegate() { return delegate; } - synchronized long getCount(final String requestType) { - return operationCount.getOrDefault(requestType, 0L); + synchronized Map getOperationsCount() { + return Map.copyOf(operationCount); } protected synchronized void trackRequest(final String requestType) { From a7b2ef88d1b4581061fb4fdfa6f15b8d815d7bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Mon, 7 Sep 2020 11:09:08 +0200 Subject: [PATCH 19/19] Minor corrections --- .../apis/clear-repositories-metering-archive.asciidoc | 4 ++-- .../apis/get-repositories-metering.asciidoc | 4 ++-- .../apis/repositories-meterings-body.asciidoc | 10 +++++----- .../repositories-metering-apis.asciidoc | 2 +- .../action/RepositoriesMeteringResponseTests.java | 8 ++++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc index 8254f16ac2867..33441ee5efa31 100644 --- a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc +++ b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc @@ -16,7 +16,7 @@ Removes the archived repositories metering information present in the cluster. [[clear-repositories-metering-archive-api-desc]] ==== {api-description-title} -You can use this API to clear the archived repositories meterings in the cluster. +You can use this API to clear the archived repositories metering information in the cluster. [[clear-repositories-metering-archive-api-path-params]] ==== {api-path-parms-title} @@ -30,6 +30,6 @@ All the nodes selective options are explained <>. [role="child_attributes"] [[clear-repositories-metering-archive-api-response-body]] ==== {api-response-body-title} -Returns the deleted archived repositories meterings. +Returns the deleted archived repositories metering information. include::{es-repo-dir}/repositories-metering-api/apis/repositories-meterings-body.asciidoc[tag=repositories-metering-body] diff --git a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc index 3485a9a4b7bec..409d74cbde295 100644 --- a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc +++ b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc @@ -1,9 +1,9 @@ [role="xpack"] [testenv="basic"] [[get-repositories-metering-api]] -=== Get repositories meterings +=== Get repositories metering information ++++ -Get repositories meterings +Get repositories metering information ++++ Returns cluster repositories metering information. diff --git a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc index 9547718c1d495..fa37bb6ba1853 100644 --- a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc +++ b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc @@ -39,11 +39,11 @@ An array of repository metering information for the node. [%collapsible%open] ===== `repository_name`:: -(string) +(string) Repository name. `repository_type`:: -(string) +(string) Repository type. `repository_location`:: @@ -86,7 +86,7 @@ The path within the bucket where the repository stores data. Bucket name. ====== `repository_ephemeral_id`:: -(string) +(string) An identifier that changes every time the repository is updated. `repository_started_at`:: @@ -100,7 +100,7 @@ Time the repository was deleted or updated. Recorded in milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch]. `archived`:: -(boolean) +(boolean) A flag that tells whether or not this object has been archived. When a repository is closed or updated the repository metering information is archived and kept for a certain period of time. This allows retrieving @@ -117,7 +117,7 @@ information deletions, i.e. deleting archived repositories metering information that we haven't observed yet. `request_counts`:: -(object) +(object) An object with the number of request performed against the repository grouped by request type. + diff --git a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc index 58f84b035ddcd..7d6a0b8da0a2c 100644 --- a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc +++ b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc @@ -9,7 +9,7 @@ You can use the following APIs to retrieve repositories metering information. This is an API used by Elastic's commercial offerings. -* <> +* <> * <> include::apis/get-repositories-metering.asciidoc[] diff --git a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java index e68aa5bf5a283..3c27541b596f2 100644 --- a/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java +++ b/x-pack/plugin/repositories-metering-api/src/test/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringResponseTests.java @@ -59,12 +59,12 @@ private RepositoriesMeteringResponse createResponse() { ClusterName clusterName = new ClusterName("test"); int nodes = randomIntBetween(1, 10); List nodeResponses = new ArrayList<>(nodes); - for (int i = 0; i < nodes; i++) { - DiscoveryNode node = new DiscoveryNode("nodeId" + i, buildNewFakeTransportAddress(), Version.CURRENT); + for (int nodeId = 0; nodeId < nodes; nodeId++) { + DiscoveryNode node = new DiscoveryNode("nodeId" + nodeId, buildNewFakeTransportAddress(), Version.CURRENT); int numberOfRepos = randomInt(10); List nodeRepoStats = new ArrayList<>(numberOfRepos); - for (int j = 0; j < numberOfRepos; j++) { + for (int clusterVersion = 0; clusterVersion < numberOfRepos; clusterVersion++) { String repoId = randomAlphaOfLength(10); String repoName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); String repoType = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); @@ -76,7 +76,7 @@ private RepositoriesMeteringResponse createResponse() { RepositoryStatsSnapshot statsSnapshot = new RepositoryStatsSnapshot( repositoryInfo, new RepositoryStats(Map.of("GET", randomLongBetween(0, 2000))), - archived ? j : RepositoryStatsSnapshot.UNKNOWN_CLUSTER_VERSION, + archived ? clusterVersion : RepositoryStatsSnapshot.UNKNOWN_CLUSTER_VERSION, archived ); nodeRepoStats.add(statsSnapshot);