License checks for archive tier (#83894)

Implements enterprise license for archive data. Relates #81210
elastic · Feb 15, 2022 · 749ad74 · 749ad74
1 parent 64929dc
commit 749ad74
Show file tree

Hide file tree

Showing 15 changed files with 612 additions and 36 deletions.
diff --git a/server/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java b/server/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.plugins;
 
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.env.Environment;
@@ -17,6 +18,7 @@
 
 import java.util.Collections;
 import java.util.Map;
+import java.util.function.Consumer;
 
 /**
  * An extension point for {@link Plugin} implementations to add custom snapshot repositories.
@@ -59,4 +61,13 @@ default Map<String, Repository.Factory> getInternalRepositories(
         return Collections.emptyMap();
     }
 
+    /**
+     * Returns a check that is run on restore. This allows plugins to prevent certain restores from happening.
+     *
+     * returns null if no check is provided
+     */
+    default Consumer<IndexMetadata> addPreRestoreCheck() {
+        return null;
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.repositories;
 
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
@@ -18,10 +19,12 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 
 /**
  * Sets up classes for Snapshot/Restore.
@@ -80,6 +83,14 @@ public RepositoriesModule(
             }
         }
 
+        List<Consumer<IndexMetadata>> preRestoreChecks = new ArrayList<>();
+        for (RepositoryPlugin repoPlugin : repoPlugins) {
+            Consumer<IndexMetadata> preRestoreCheck = repoPlugin.addPreRestoreCheck();
+            if (preRestoreCheck != null) {
+                preRestoreChecks.add(preRestoreCheck);
+            }
+        }
+
         Settings settings = env.settings();
         Map<String, Repository.Factory> repositoryTypes = Collections.unmodifiableMap(factories);
         Map<String, Repository.Factory> internalRepositoryTypes = Collections.unmodifiableMap(internalFactories);
@@ -89,7 +100,8 @@ public RepositoriesModule(
             transportService,
             repositoryTypes,
             internalRepositoryTypes,
-            transportService.getThreadPool()
+            transportService.getThreadPool(),
+            preRestoreChecks
         );
     }
 

diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -56,6 +56,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -97,13 +98,16 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C
     private volatile Map<String, Repository> repositories = Collections.emptyMap();
     private final RepositoriesStatsArchive repositoriesStatsArchive;
 
+    private final List<Consumer<IndexMetadata>> preRestoreChecks;
+
     public RepositoriesService(
         Settings settings,
         ClusterService clusterService,
         TransportService transportService,
         Map<String, Repository.Factory> typesRegistry,
         Map<String, Repository.Factory> internalTypesRegistry,
-        ThreadPool threadPool
+        ThreadPool threadPool,
+        List<Consumer<IndexMetadata>> preRestoreChecks
     ) {
         this.typesRegistry = typesRegistry;
         this.internalTypesRegistry = internalTypesRegistry;
@@ -122,6 +126,7 @@ public RepositoriesService(
             REPOSITORIES_STATS_ARCHIVE_MAX_ARCHIVED_STATS.get(settings),
             threadPool::relativeTimeInMillis
         );
+        this.preRestoreChecks = preRestoreChecks;
     }
 
     /**
@@ -776,6 +781,10 @@ private static RepositoryConflictException newRepositoryConflictException(String
         );
     }
 
+    public List<Consumer<IndexMetadata>> getPreRestoreChecks() {
+        return preRestoreChecks;
+    }
+
     @Override
     protected void doStart() {
 

diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -1289,9 +1289,11 @@ public ClusterState execute(ClusterState currentState) {
             final String localNodeId = clusterService.state().nodes().getLocalNodeId();
             for (Map.Entry<String, IndexId> indexEntry : indicesToRestore.entrySet()) {
                 final IndexId index = indexEntry.getValue();
+                final IndexMetadata originalIndexMetadata = metadata.index(index.getName());
+                repositoriesService.getPreRestoreChecks().forEach(check -> check.accept(originalIndexMetadata));
                 IndexMetadata snapshotIndexMetadata = updateIndexSettings(
                     snapshot,
-                    metadata.index(index.getName()),
+                    originalIndexMetadata,
                     request.indexSettings(),
                     request.ignoreIndexSettings()
                 );
@@ -1591,39 +1593,40 @@ private IndexMetadata convertLegacyIndex(IndexMetadata snapshotIndexMetadata, Cl
         if (snapshotIndexMetadata.getCreationVersion().before(Version.fromString("5.0.0"))) {
             throw new IllegalArgumentException("can't restore an index created before version 5.0.0");
         }
+        IndexMetadata.Builder convertedIndexMetadata = IndexMetadata.builder(snapshotIndexMetadata);
         MappingMetadata mappingMetadata = snapshotIndexMetadata.mapping();
-        Map<String, Object> loadedMappingSource = mappingMetadata.rawSourceAsMap();
-
-        // store old mapping under _meta/legacy_mappings
-        Map<String, Object> legacyMapping = new LinkedHashMap<>();
-        boolean sourceOnlySnapshot = snapshotIndexMetadata.getSettings().getAsBoolean("index.source_only", false);
-        if (sourceOnlySnapshot) {
-            // actual mapping is under "_meta" (but strip type first)
-            Object sourceOnlyMeta = mappingMetadata.sourceAsMap().get("_meta");
-            if (sourceOnlyMeta instanceof Map<?, ?> sourceOnlyMetaMap) {
-                legacyMapping.put("legacy_mappings", sourceOnlyMetaMap);
+        if (mappingMetadata != null) {
+            Map<String, Object> loadedMappingSource = mappingMetadata.rawSourceAsMap();
+
+            // store old mapping under _meta/legacy_mappings
+            Map<String, Object> legacyMapping = new LinkedHashMap<>();
+            boolean sourceOnlySnapshot = snapshotIndexMetadata.getSettings().getAsBoolean("index.source_only", false);
+            if (sourceOnlySnapshot) {
+                // actual mapping is under "_meta" (but strip type first)
+                Object sourceOnlyMeta = mappingMetadata.sourceAsMap().get("_meta");
+                if (sourceOnlyMeta instanceof Map<?, ?> sourceOnlyMetaMap) {
+                    legacyMapping.put("legacy_mappings", sourceOnlyMetaMap);
+                }
+            } else {
+                legacyMapping.put("legacy_mappings", loadedMappingSource);
             }
-        } else {
-            legacyMapping.put("legacy_mappings", loadedMappingSource);
-        }
 
-        Map<String, Object> newMappingSource = new LinkedHashMap<>();
-        newMappingSource.put("_meta", legacyMapping);
+            Map<String, Object> newMappingSource = new LinkedHashMap<>();
+            newMappingSource.put("_meta", legacyMapping);
 
-        Map<String, Object> newMapping = new LinkedHashMap<>();
-        newMapping.put(mappingMetadata.type(), newMappingSource);
+            Map<String, Object> newMapping = new LinkedHashMap<>();
+            newMapping.put(mappingMetadata.type(), newMappingSource);
+
+            convertedIndexMetadata.putMapping(new MappingMetadata(mappingMetadata.type(), newMapping));
+        }
+
+        convertedIndexMetadata.settings(
+            Settings.builder()
+                .put(snapshotIndexMetadata.getSettings())
+                .put(IndexMetadata.SETTING_INDEX_VERSION_COMPATIBILITY.getKey(), clusterState.getNodes().getSmallestNonClientNodeVersion())
+        );
         // TODO: _routing? Perhaps we don't need to obey any routing here as stuff is read-only anyway and get API will be disabled
-        return IndexMetadata.builder(snapshotIndexMetadata)
-            .putMapping(new MappingMetadata(mappingMetadata.type(), newMapping))
-            .settings(
-                Settings.builder()
-                    .put(snapshotIndexMetadata.getSettings())
-                    .put(
-                        IndexMetadata.SETTING_INDEX_VERSION_COMPATIBILITY.getKey(),
-                        clusterState.getNodes().getSmallestNonClientNodeVersion()
-                    )
-            )
-            .build();
+        return convertedIndexMetadata.build();
     }
 
     private static IndexMetadata.Builder restoreToCreateNewIndex(IndexMetadata snapshotIndexMetadata, String renamedIndexName) {

diff --git a/.../java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/.../java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -525,7 +525,8 @@ private IndicesClusterStateService createIndicesClusterStateService(
             transportService,
             Collections.emptyMap(),
             Collections.emptyMap(),
-            threadPool
+            threadPool,
+            List.of()
         );
         final PeerRecoveryTargetService recoveryTargetService = new PeerRecoveryTargetService(
             threadPool,

diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
@@ -93,7 +93,8 @@ public void setUp() throws Exception {
             transportService,
             typesRegistry,
             typesRegistry,
-            threadPool
+            threadPool,
+            List.of()
         );
         repositoriesService.start();
     }

diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -1715,7 +1715,8 @@ protected void assertSnapshotOrGenericThread() {
                         }
                     ),
                     emptyMap(),
-                    threadPool
+                    threadPool,
+                    List.of()
                 );
                 final ActionFilters actionFilters = new ActionFilters(emptySet());
                 snapshotsService = new SnapshotsService(

diff --git a/...lugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/...lugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.coordination.ElectionStrategy;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
@@ -104,6 +105,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.BiConsumer;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
 import java.util.function.Predicate;
@@ -568,6 +570,15 @@ public Map<String, Repository.Factory> getInternalRepositories(
         return internalRepositories;
     }
 
+    @Override
+    public Consumer<IndexMetadata> addPreRestoreCheck() {
+        List<Consumer<IndexMetadata>> checks = filterPlugins(RepositoryPlugin.class).stream()
+            .map(RepositoryPlugin::addPreRestoreCheck)
+            .filter(Objects::nonNull)
+            .collect(Collectors.toList());
+        return checks.isEmpty() ? null : imd -> checks.forEach(c -> c.accept(imd));
+    }
+
     @Override
     public void close() throws IOException {
         IOUtils.close(plugins);

diff --git a/x-pack/plugin/old-lucene-versions/build.gradle b/x-pack/plugin/old-lucene-versions/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
@@ -11,6 +12,7 @@ archivesBaseName = 'x-pack-old-lucene-versions'
 
 dependencies {
   compileOnly project(path: xpackModule('core'))
+  internalClusterTestImplementation(testArtifact(project(xpackModule('core'))))
 }
 
 addQaCheckDependencies()