From 95ba9c4930b507ea885c94a264e66c4f3a40ee2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 6 Nov 2024 22:07:20 +0100 Subject: [PATCH 01/24] WIP --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 6 +- .../bwc/codecs/lucene87/BWCLucene87Codec.java | 153 ++++++++++++++++++ .../services/org.apache.lucene.codecs.Codec | 1 + 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 3ed8fc26ac937..1f367147a582a 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; +import org.apache.lucene.backward_codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldsConsumer; @@ -26,6 +27,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec; import java.io.IOException; import java.util.ArrayList; @@ -118,7 +120,9 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) { } public static SegmentInfo wrap(SegmentInfo segmentInfo) { - final Codec codec = segmentInfo.getCodec(); + // special handling for Lucene87Codec (which is currently bundled with Lucene) + // Use BWCLucene87Codec instead as that one extends BWCCodec (similar to all other older codecs) + final Codec codec = segmentInfo.getCodec() instanceof Lucene87Codec ? new BWCLucene87Codec() : segmentInfo.getCodec(); final SegmentInfo segmentInfo1 = new SegmentInfo( segmentInfo.dir, // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java new file mode 100644 index 0000000000000..7d36811a30626 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene87; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; +import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + +public class BWCLucene87Codec extends BWCCodec { + + private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86PointsFormat(); + private final PostingsFormat defaultFormat; + + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return BWCLucene87Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return BWCLucene87Codec.this.getDocValuesFormatForField(field); + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + + /** Instantiates a new codec. */ + public BWCLucene87Codec() { + super("BWCLucene87Codec"); + this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION); + this.defaultFormat = new Lucene84PostingsFormat(); + this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final KnnVectorsFormat knnVectorsFormat() { + return KnnVectorsFormat.EMPTY; + } + + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene84". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene80". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final DocValuesFormat defaultDVFormat; + + private final NormsFormat normsFormat = new Lucene80NormsFormat(); + + @Override + public NormsFormat normsFormat() { + return normsFormat; + } + +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 0215e9f7ca4ab..e94008f06ea29 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -5,6 +5,7 @@ # 2.0. # +org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec From d2817fa72dd40a75b04f03a6f1130d4c3037419c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 21 Nov 2024 11:49:47 +0100 Subject: [PATCH 02/24] Trying to add new qa test project for 7x archival indices --- .../repository-old-versions-7x/build.gradle | 138 ++ .../oldrepos/DocValueOnlyFieldsIT.java | 202 +++ .../elasticsearch/oldrepos/OldMappingsIT.java | 361 ++++ .../oldrepos/OldRepositoryAccessIT.java | 512 ++++++ .../org/elasticsearch/oldrepos/custom.json | 22 + .../org/elasticsearch/oldrepos/filebeat.json | 681 +++++++ .../org/elasticsearch/oldrepos/nested.json | 7 + .../elasticsearch/oldrepos/winlogbeat.json | 1612 +++++++++++++++++ 8 files changed, 3535 insertions(+) create mode 100644 x-pack/qa/repository-old-versions-7x/build.gradle create mode 100644 x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java create mode 100644 x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java create mode 100644 x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java create mode 100644 x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/custom.json create mode 100644 x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/filebeat.json create mode 100644 x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/nested.json create mode 100644 x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/winlogbeat.json diff --git a/x-pack/qa/repository-old-versions-7x/build.gradle b/x-pack/qa/repository-old-versions-7x/build.gradle new file mode 100644 index 0000000000000..1c0ce552795d1 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/build.gradle @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import org.elasticsearch.gradle.Architecture +import org.elasticsearch.gradle.OS +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.internal.BwcVersions +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.AntFixture +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask +import org.elasticsearch.gradle.transform.UnzipTransform +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + +apply plugin: 'elasticsearch.jdk-download' +apply plugin: 'elasticsearch.internal-testclusters' +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.rest-resources' + +jdks { + legacy { + vendor = 'adoptium' + version = '8u302+b08' + platform = OS.current().name().toLowerCase() + architecture = Architecture.current().name().toLowerCase() + } +} + +restResources { + restApi { + include '_common', 'search' + } + restTests { + includeCore 'search/390_doc_values_search.yml' + } +} + +if (OS.current() == OS.MAC && Architecture.current() == Architecture.AARCH64) { + jdks.legacy.vendor = 'zulu' + jdks.legacy.distributionVersion = '8.56.0.23' +} + +if (OS.current() == OS.WINDOWS) { + logger.warn("Disabling repository-old-versions tests because we can't get the pid file on windows") +} else { + int currentMajorVersion = org.elasticsearch.gradle.VersionProperties.elasticsearchVersion.major + for (String versionString : ['7.17.25']) { + Version version = Version.fromString(versionString) + String versionNoDots = version.toString().replace('.', '_') +// String configName = "es${versionNoDots}" + +// def config = configurations.create(configName) +// config.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); +// dependencies.add(configName, artifact) + + String repoLocation = "${buildDir}/cluster/shared/repo/${versionNoDots}" + String clusterName = versionNoDots + + def testClusterProvider = testClusters.register(clusterName) { + testDistribution = 'DEFAULT' + numberOfNodes = 2 + versions = [project.version, project.version] // to test full cluster restart + + setting 'path.repo', repoLocation, IGNORE_VALUE + setting 'xpack.license.self_generated.type', 'trial' + + setting 'xpack.security.enabled', 'true' + user username: 'admin', password: 'admin-password', role: 'superuser' + + setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' + setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' + } + + def oldCluster = testClusters.register("oldES${versionNoDots}Cluster") { + testDistribution = 'DEFAULT' + numberOfNodes = 1 + versions = [versionString] + setting 'path.repo', repoLocation, IGNORE_VALUE + setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.security.enabled', 'true' + user username: 'admin', password: 'admin-password', role: 'superuser' + setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' + setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' + } + + tasks.register("javaRestTestBeforeRestart#${versionNoDots}", StandaloneRestIntegTestTask) { + useCluster testClusterProvider + useCluster testClusters.named("oldES${versionNoDots}Cluster") + doFirst { + delete(repoLocation) + mkdir(repoLocation) + } + systemProperty 'tests.after_restart', 'false' + } + + tasks.register("javaRestTestAfterRestart#${versionNoDots}", StandaloneRestIntegTestTask) { + useCluster testClusterProvider + dependsOn "javaRestTestBeforeRestart#${versionNoDots}" + useCluster testClusters.named("oldES${versionNoDots}Cluster") + systemProperty 'tests.after_restart', 'true' + + doFirst { + testClusterProvider.get().goToNextVersion() + } + } + + tasks.matching { it.name.startsWith("javaRestTest") && it.name.endsWith(versionNoDots) }.configureEach { + doFirst { + def oldClusterPortProvider = testClusters.named("oldES${versionNoDots}Cluster").map { cluster -> + def local = cluster.allHttpSocketURI.find { it.startsWith("127.0.0.1")} + return local.substring(local.lastIndexOf(':') + 1) + } + it.nonInputProperties.systemProperty "tests.es.port", "${-> oldClusterPortProvider.get()}" + println "oldClusterPortProvider " + oldClusterPortProvider.get() + println "allRemoteAccessPortURI " + testClusters.named("oldES${versionNoDots}Cluster").get().allRemoteAccessPortURI + println "allHttpSocketURI " + testClusters.named("oldES${versionNoDots}Cluster").get().allHttpSocketURI + println "httpSocketURI " + testClusters.named("oldES${versionNoDots}Cluster").get().httpSocketURI + } + it.nonInputProperties.systemProperty "tests.repo.location", repoLocation + it.systemProperty "tests.es.version", version.toString() + + /* Use a closure on the string to delay evaluation until right before we + * run the integration tests so that we can be sure that the file is + * ready. */ + it.nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusterProvider.get().allHttpSocketURI.join(",")}") + it.nonInputProperties.systemProperty('tests.clustername', "${-> testClusterProvider.get().getName()}") + } + + tasks.named("check").configure { + dependsOn "javaRestTestAfterRestart#${versionNoDots}" + } + } +} + diff --git a/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java b/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java new file mode 100644 index 0000000000000..968262448c87e --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.http.HttpHost; +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.junit.Before; + +import java.io.IOException; + +/** + * Tests doc-value-based searches against indices imported from clusters older than N-1. + * We reuse the YAML tests in search/390_doc_values_search.yml but have to do the setup + * manually here as the setup is done on the old cluster for which we have to use the + * low-level REST client instead of the YAML set up that only knows how to talk to + * newer ES versions. + * + * We mimic the setup in search/390_doc_values_search.yml here, but adapt it to work + * against older version clusters. + */ +public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase { + + final Version oldVersion = Version.fromString(System.getProperty("tests.es.version")); + static boolean setupDone; + + public DocValueOnlyFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Override + protected boolean skipSetupSections() { + // setup in the YAML file is replaced by the method below + return true; + } + + @Before + public void setupIndex() throws IOException { + final boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); + if (afterRestart) { + return; + } + + // The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only + // initialized later, so we do it when running the first test + if (setupDone) { + return; + } + + setupDone = true; + + String repoLocation = PathUtils.get(System.getProperty("tests.repo.location")) + .resolve(RandomizedTest.getContext().getTargetClass().getName()) + .toString(); + + String indexName = "test"; + String repoName = "doc_values_repo"; + String snapshotName = "snap"; + String[] basicTypes = new String[] { + "byte", + "double", + "float", + "half_float", + "integer", + "long", + "short", + "boolean", + "keyword", + "ip", + "geo_point" }; // date is manually added as it need further configuration + + int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); + try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { + Request createIndex = new Request("PUT", "/" + indexName); + int numberOfShards = randomIntBetween(1, 3); + + boolean multiTypes = oldVersion.before(Version.V_7_0_0); + + XContentBuilder settingsBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject("settings") + .field("index.number_of_shards", numberOfShards) + .endObject() + .startObject("mappings"); + if (multiTypes) { + settingsBuilder.startObject("doc"); + } + settingsBuilder.field("dynamic", false).startObject("properties"); + for (String type : basicTypes) { + settingsBuilder.startObject(type).field("type", type).endObject(); + } + settingsBuilder.startObject("date").field("type", "date").field("format", "yyyy/MM/dd").endObject(); + if (multiTypes) { + settingsBuilder.endObject(); + } + settingsBuilder.endObject().endObject().endObject(); + + createIndex.setJsonEntity(Strings.toString(settingsBuilder)); + assertOK(oldEs.performRequest(createIndex)); + + Request doc1 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "1"); + doc1.addParameter("refresh", "true"); + XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() + .startObject() + .field("byte", 1) + .field("double", 1.0) + .field("float", 1.0) + .field("half_float", 1.0) + .field("integer", 1) + .field("long", 1) + .field("short", 1) + .field("date", "2017/01/01") + .field("keyword", "key1") + .field("boolean", false) + .field("ip", "192.168.0.1") + .array("geo_point", 13.5, 34.89) + .endObject(); + doc1.setJsonEntity(Strings.toString(bodyDoc1)); + assertOK(oldEs.performRequest(doc1)); + + Request doc2 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "2"); + doc2.addParameter("refresh", "true"); + XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() + .startObject() + .field("byte", 2) + .field("double", 2.0) + .field("float", 2.0) + .field("half_float", 2.0) + .field("integer", 2) + .field("long", 2) + .field("short", 2) + .field("date", "2017/01/02") + .field("keyword", "key2") + .field("boolean", true) + .field("ip", "192.168.0.2") + .array("geo_point", -63.24, 31.0) + .endObject(); + doc2.setJsonEntity(Strings.toString(bodyDoc2)); + assertOK(oldEs.performRequest(doc2)); + + // register repo on old ES and take snapshot + Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(oldEs.performRequest(createRepoRequest)); + + Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); + assertOK(oldEs.performRequest(createSnapshotRequest)); + } + + // register repo on new ES and restore snapshot + Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest2.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(client().performRequest(createRepoRequest2)); + + final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); + createRestoreRequest.addParameter("wait_for_completion", "true"); + createRestoreRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); + assertOK(client().performRequest(createRestoreRequest)); + } +} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java new file mode 100644 index 0000000000000..67dbdec6b8399 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java @@ -0,0 +1,361 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos; + +import com.carrotsearch.randomizedtesting.RandomizedTest; + +import org.apache.http.HttpHost; +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; + +public class OldMappingsIT extends ESRestTestCase { + + static final Version oldVersion = Version.fromString(System.getProperty("tests.es.version")); + + static boolean setupDone; + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Before + public void setupIndex() throws IOException { + final boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); + if (afterRestart) { + return; + } + + // The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only + // initialized later, so we do it when running the first test + if (setupDone) { + return; + } + + setupDone = true; + + String repoLocation = PathUtils.get(System.getProperty("tests.repo.location")) + .resolve(RandomizedTest.getContext().getTargetClass().getName()) + .toString(); + + String repoName = "old_mappings_repo"; + String snapshotName = "snap"; + List indices; + if (oldVersion.before(Version.fromString("6.0.0"))) { + indices = Arrays.asList("filebeat", "winlogbeat", "custom", "nested"); + } else { + indices = Arrays.asList("filebeat", "custom", "nested"); + } + + int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); + try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { + + assertOK(oldEs.performRequest(createIndex("filebeat", "filebeat.json"))); + if (oldVersion.before(Version.fromString("6.0.0"))) { + assertOK(oldEs.performRequest(createIndex("winlogbeat", "winlogbeat.json"))); + } + assertOK(oldEs.performRequest(createIndex("custom", "custom.json"))); + assertOK(oldEs.performRequest(createIndex("nested", "nested.json"))); + + Request doc1 = new Request("PUT", "/" + "custom" + "/" + "doc" + "/" + "1"); + doc1.addParameter("refresh", "true"); + XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() + .startObject() + .startObject("apache2") + .startObject("access") + .field("url", "myurl1") + .field("agent", "agent1") + .endObject() + .endObject() + .endObject(); + doc1.setJsonEntity(Strings.toString(bodyDoc1)); + assertOK(oldEs.performRequest(doc1)); + + Request doc2 = new Request("PUT", "/" + "custom" + "/" + "doc" + "/" + "2"); + doc2.addParameter("refresh", "true"); + XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() + .startObject() + .startObject("apache2") + .startObject("access") + .field("url", "myurl2") + .field("agent", "agent2 agent2") + .endObject() + .endObject() + .field("completion", "some_value") + .endObject(); + doc2.setJsonEntity(Strings.toString(bodyDoc2)); + assertOK(oldEs.performRequest(doc2)); + + Request doc3 = new Request("PUT", "/" + "nested" + "/" + "doc" + "/" + "1"); + doc3.addParameter("refresh", "true"); + XContentBuilder bodyDoc3 = XContentFactory.jsonBuilder() + .startObject() + .field("group", "fans") + .startArray("user") + .startObject() + .field("first", "John") + .field("last", "Smith") + .endObject() + .startObject() + .field("first", "Alice") + .field("last", "White") + .endObject() + .endArray() + .endObject(); + doc3.setJsonEntity(Strings.toString(bodyDoc3)); + assertOK(oldEs.performRequest(doc3)); + + // register repo on old ES and take snapshot + Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(oldEs.performRequest(createRepoRequest)); + + Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); + assertOK(oldEs.performRequest(createSnapshotRequest)); + } + + // register repo on new ES and restore snapshot + Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest2.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(client().performRequest(createRepoRequest2)); + + final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); + createRestoreRequest.addParameter("wait_for_completion", "true"); + createRestoreRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); + createRestoreRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + assertOK(client().performRequest(createRestoreRequest)); + } + + private Request createIndex(String indexName, String file) throws IOException { + Request createIndex = new Request("PUT", "/" + indexName); + int numberOfShards = randomIntBetween(1, 3); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("settings") + .field("index.number_of_shards", numberOfShards) + .endObject() + .startObject("mappings"); + builder.rawValue(OldMappingsIT.class.getResourceAsStream(file), XContentType.JSON); + builder.endObject().endObject(); + + createIndex.setJsonEntity(Strings.toString(builder)); + return createIndex; + } + + public void testMappingOk() throws IOException { + Request mappingRequest = new Request("GET", "/" + "filebeat" + "/_mapping"); + Map mapping = entityAsMap(client().performRequest(mappingRequest)); + assertNotNull(XContentMapValues.extractValue(mapping, "filebeat", "mappings", "properties", "apache2")); + + if (oldVersion.before(Version.fromString("6.0.0"))) { + mappingRequest = new Request("GET", "/" + "winlogbeat" + "/_mapping"); + mapping = entityAsMap(client().performRequest(mappingRequest)); + assertNotNull(XContentMapValues.extractValue(mapping, "winlogbeat", "mappings", "properties", "message")); + } + } + + public void testSearchKeyword() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("match") + .startObject("apache2.access.url") + .field("query", "myurl2") + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + Map response = entityAsMap(client().performRequest(search)); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + } + + public void testSearchOnPlaceHolderField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("match") + .startObject("completion") + .field("query", "some-agent") + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + ResponseException re = expectThrows(ResponseException.class, () -> entityAsMap(client().performRequest(search))); + assertThat( + re.getMessage(), + containsString("Field [completion] of type [completion] in legacy index does not support match queries") + ); + } + + public void testAggregationOnPlaceholderField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("aggs") + .startObject("agents") + .startObject("terms") + .field("field", "completion") + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + ResponseException re = expectThrows(ResponseException.class, () -> entityAsMap(client().performRequest(search))); + assertThat(re.getMessage(), containsString("can't run aggregation or sorts on field type completion of legacy index")); + } + + public void testConstantScoringOnTextField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("match") + .startObject("apache2.access.agent") + .field("query", "agent2") + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + Map response = entityAsMap(client().performRequest(search)); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + @SuppressWarnings("unchecked") + Map hit = (Map) hits.get(0); + assertThat(hit, hasKey("_score")); + assertEquals(1.0d, (double) hit.get("_score"), 0.01d); + } + + public void testFieldsExistQueryOnTextField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("exists") + .field("field", "apache2.access.agent") + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + Map response = entityAsMap(client().performRequest(search)); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(2)); + } + + public void testSearchFieldsOnPlaceholderField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("match") + .startObject("apache2.access.url") + .field("query", "myurl2") + .endObject() + .endObject() + .endObject() + .startArray("fields") + .value("completion") + .endArray() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + Map response = entityAsMap(client().performRequest(search)); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + logger.info(hits); + Map fields = (Map) (XContentMapValues.extractValue("fields", (Map) hits.get(0))); + assertEquals(List.of("some_value"), fields.get("completion")); + } + + public void testNestedDocuments() throws IOException { + Request search = new Request("POST", "/" + "nested" + "/_search"); + Map response = entityAsMap(client().performRequest(search)); + logger.info(response); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + Map source = (Map) (XContentMapValues.extractValue("_source", (Map) hits.get(0))); + assertEquals("fans", source.get("group")); + + search = new Request("POST", "/" + "nested" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("nested") + .field("path", "user") + .startObject("query") + .startObject("bool") + .startArray("must") + .startObject() + .startObject("match") + .field("user.first", "Alice") + .endObject() + .endObject() + .startObject() + .startObject("match") + .field("user.last", "White") + .endObject() + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + response = entityAsMap(client().performRequest(search)); + logger.info(response); + hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + source = (Map) (XContentMapValues.extractValue("_source", (Map) hits.get(0))); + assertEquals("fans", source.get("group")); + } + +} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java new file mode 100644 index 0000000000000..30ec6630b9618 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -0,0 +1,512 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos; + +import org.apache.http.HttpHost; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchResponseUtils; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; + +public class OldRepositoryAccessIT extends ESRestTestCase { + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + public void testOldRepoAccess() throws IOException { + runTest(false); + } + + public void testOldSourceOnlyRepoAccess() throws IOException { + runTest(true); + } + + public void runTest(boolean sourceOnlyRepository) throws IOException { + boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); + String repoLocation = System.getProperty("tests.repo.location"); + repoLocation = PathUtils.get(repoLocation).resolve("source_only_" + sourceOnlyRepository).toString(); + Version oldVersion = Version.fromString(System.getProperty("tests.es.version")); + assumeTrue( + "source only repositories only supported since ES 6.5.0", + sourceOnlyRepository == false || oldVersion.onOrAfter(Version.fromString("6.5.0")) + ); + + assertThat("Index version should be added to archive tests", oldVersion, lessThan(Version.V_8_10_0)); + IndexVersion indexVersion = IndexVersion.fromId(oldVersion.id); + + int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); + String indexName; + if (sourceOnlyRepository) { + indexName = "source_only_test_index"; + } else { + indexName = "test_index"; + } + int numDocs = 10; + int extraDocs = 1; + final Set expectedIds = new HashSet<>(); + try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { + if (afterRestart == false) { + beforeRestart( + sourceOnlyRepository, + repoLocation, + oldVersion, + indexVersion, + numDocs, + extraDocs, + expectedIds, + oldEs, + indexName + ); + } else { + afterRestart(indexName); + } + } + } + + private void afterRestart(String indexName) throws IOException { + ensureGreen("restored_" + indexName); + ensureGreen("mounted_full_copy_" + indexName); + ensureGreen("mounted_shared_cache_" + indexName); + } + + private void beforeRestart( + boolean sourceOnlyRepository, + String repoLocation, + Version oldVersion, + IndexVersion indexVersion, + int numDocs, + int extraDocs, + Set expectedIds, + RestClient oldEs, + String indexName + ) throws IOException { + String repoName = "repo_" + indexName; + String snapshotName = "snap_" + indexName; + Request createIndex = new Request("PUT", "/" + indexName); + int numberOfShards = randomIntBetween(1, 3); + + XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject().startObject("settings"); + settingsBuilder.field("index.number_of_shards", numberOfShards); + + // 6.5.0 started using soft-deletes, but it was only enabled by default on 7.0 + if (oldVersion.onOrAfter(Version.fromString("6.5.0")) && oldVersion.before(Version.fromString("7.0.0")) && randomBoolean()) { + settingsBuilder.field("index.soft_deletes.enabled", true); + } + + settingsBuilder.endObject().endObject(); + + createIndex.setJsonEntity(Strings.toString(settingsBuilder)); + assertOK(oldEs.performRequest(createIndex)); + + for (int i = 0; i < numDocs + extraDocs; i++) { + String id = "testdoc" + i; + expectedIds.add(id); + // use multiple types for ES versions < 6.0.0 + String type = getType(oldVersion, id); + Request doc = new Request("PUT", "/" + indexName + "/" + type + "/" + id); + doc.addParameter("refresh", "true"); + doc.setJsonEntity(sourceForDoc(i)); + assertOK(oldEs.performRequest(doc)); + } + + for (int i = 0; i < extraDocs; i++) { + String id = randomFrom(expectedIds); + expectedIds.remove(id); + String type = getType(oldVersion, id); + Request doc = new Request("DELETE", "/" + indexName + "/" + type + "/" + id); + doc.addParameter("refresh", "true"); + oldEs.performRequest(doc); + } + + // register repo on old ES and take snapshot + Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest.setJsonEntity(sourceOnlyRepository ? Strings.format(""" + {"type":"source","settings":{"location":"%s","delegate_type":"fs"}} + """, repoLocation) : Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(oldEs.performRequest(createRepoRequest)); + + Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); + assertOK(oldEs.performRequest(createSnapshotRequest)); + + // register repo on new ES + Settings.Builder repoSettingsBuilder = Settings.builder().put("location", repoLocation); + if (sourceOnlyRepository) { + repoSettingsBuilder.put("delegate_type", "fs"); + } + Request createRepo = new Request("PUT", "/_snapshot/" + repoName); + createRepo.setJsonEntity( + Strings.toString( + new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).type(sourceOnlyRepository ? "source" : "fs") + .settings(repoSettingsBuilder.build()) + ) + ); + assertAcknowledged(client().performRequest(createRepo)); + + // list snapshots on new ES + Request getSnaps = new Request("GET", "/_snapshot/" + repoName + "/_all"); + Response getResponse = client().performRequest(getSnaps); + ObjectPath getResp = ObjectPath.createFromResponse(getResponse); + assertThat(getResp.evaluate("total"), equalTo(1)); + assertThat(getResp.evaluate("snapshots.0.snapshot"), equalTo(snapshotName)); + assertThat(getResp.evaluate("snapshots.0.repository"), equalTo(repoName)); + assertThat(getResp.evaluate("snapshots.0.indices"), contains(indexName)); + assertThat(getResp.evaluate("snapshots.0.state"), equalTo(SnapshotState.SUCCESS.toString())); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.successful")); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.total")); + assertEquals(0, (int) getResp.evaluate("snapshots.0.shards.failed")); + assertEquals(indexVersion.toReleaseVersion(), getResp.evaluate("snapshots.0.version")); + + // list specific snapshot on new ES + getSnaps = new Request("GET", "/_snapshot/" + repoName + "/" + snapshotName); + getResponse = client().performRequest(getSnaps); + getResp = ObjectPath.createFromResponse(getResponse); + assertThat(getResp.evaluate("total"), equalTo(1)); + assertThat(getResp.evaluate("snapshots.0.snapshot"), equalTo(snapshotName)); + assertThat(getResp.evaluate("snapshots.0.repository"), equalTo(repoName)); + assertThat(getResp.evaluate("snapshots.0.indices"), contains(indexName)); + assertThat(getResp.evaluate("snapshots.0.state"), equalTo(SnapshotState.SUCCESS.toString())); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.successful")); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.total")); + assertEquals(0, (int) getResp.evaluate("snapshots.0.shards.failed")); + assertEquals(indexVersion.toReleaseVersion(), getResp.evaluate("snapshots.0.version")); + + // list advanced snapshot info on new ES + getSnaps = new Request("GET", "/_snapshot/" + repoName + "/" + snapshotName + "/_status"); + getResponse = client().performRequest(getSnaps); + getResp = ObjectPath.createFromResponse(getResponse); + assertThat(((List) getResp.evaluate("snapshots")).size(), equalTo(1)); + assertThat(getResp.evaluate("snapshots.0.snapshot"), equalTo(snapshotName)); + assertThat(getResp.evaluate("snapshots.0.repository"), equalTo(repoName)); + assertThat(((Map) getResp.evaluate("snapshots.0.indices")).keySet(), contains(indexName)); + assertThat(getResp.evaluate("snapshots.0.state"), equalTo(SnapshotState.SUCCESS.toString())); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards_stats.done")); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards_stats.total")); + assertEquals(0, (int) getResp.evaluate("snapshots.0.shards_stats.failed")); + assertThat(getResp.evaluate("snapshots.0.stats.total.size_in_bytes"), greaterThan(0)); + assertThat(getResp.evaluate("snapshots.0.stats.total.file_count"), greaterThan(0)); + + // restore / mount and check whether searches work + restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); + + // close indices + closeIndex(client(), "restored_" + indexName); + closeIndex(client(), "mounted_full_copy_" + indexName); + closeIndex(client(), "mounted_shared_cache_" + indexName); + + // restore / mount again + restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); + } + + private String getType(Version oldVersion, String id) { + return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Math.abs(Murmur3HashFunction.hash(id) % 2) : 0); + } + + private static String sourceForDoc(int i) { + return "{\"test\":\"test" + i + "\",\"val\":" + i + ",\"create_date\":\"2020-01-" + Strings.format("%02d", i + 1) + "\"}"; + } + + private void restoreMountAndVerify( + int numDocs, + Set expectedIds, + int numberOfShards, + boolean sourceOnlyRepository, + Version oldVersion, + String indexName, + String repoName, + String snapshotName + ) throws IOException { + // restore index + Request restoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); + restoreRequest.setJsonEntity( + Strings.toString( + new RestoreSnapshotRequest(TEST_REQUEST_TIMEOUT).indices(indexName).renamePattern("(.+)").renameReplacement("restored_$1") + ) + ); + restoreRequest.addParameter("wait_for_completion", "true"); + Response restoreResponse = client().performRequest(restoreRequest); + ObjectPath restore = ObjectPath.createFromResponse(restoreResponse); + assertEquals(numberOfShards, (int) restore.evaluate("snapshot.shards.total")); + assertEquals(numberOfShards, (int) restore.evaluate("snapshot.shards.successful")); + + ensureGreen("restored_" + indexName); + + String restoredIndex = "restored_" + indexName; + var response = responseAsMap(client().performRequest(new Request("GET", "/" + restoredIndex + "/_mapping"))); + Map mapping = ObjectPath.evaluate(response, restoredIndex + ".mappings"); + logger.info("mapping for {}: {}", restoredIndex, mapping); + assertThat(mapping, hasKey("_meta")); + assertThat(mapping.get("_meta"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map meta = (Map) mapping.get("_meta"); + assertThat(meta, hasKey("legacy_mappings")); + assertThat(meta.get("legacy_mappings"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map legacyMappings = (Map) meta.get("legacy_mappings"); + assertThat(legacyMappings.keySet(), not(empty())); + for (Map.Entry entry : legacyMappings.entrySet()) { + String type = entry.getKey(); + assertThat(type, startsWith("doc")); + assertThat(entry.getValue(), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map legacyMapping = (Map) entry.getValue(); + assertThat(legacyMapping, hasKey("properties")); + assertThat(legacyMapping.get("properties"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map propertiesMapping = (Map) legacyMapping.get("properties"); + assertThat(propertiesMapping, hasKey("val")); + assertThat(propertiesMapping.get("val"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map valMapping = (Map) propertiesMapping.get("val"); + assertThat(valMapping, hasKey("type")); + assertEquals("long", valMapping.get("type")); + } + + // run a search against the index + assertDocs("restored_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); + + // mount as full copy searchable snapshot + Request mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); + mountRequest.setJsonEntity( + "{\"index\": \"" + + indexName + + "\",\"renamed_index\": \"mounted_full_copy_" + + indexName + + "\",\"index_settings\": {\"index.number_of_replicas\": 1}}" + ); + mountRequest.addParameter("wait_for_completion", "true"); + ObjectPath mountResponse = ObjectPath.createFromResponse(client().performRequest(mountRequest)); + assertNotNull(mountResponse.evaluate("snapshot")); + assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.total")); + assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.successful")); + + ensureGreen("mounted_full_copy_" + indexName); + + // run a search against the index + assertDocs("mounted_full_copy_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); + + // mount as shared cache searchable snapshot + mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); + mountRequest.setJsonEntity("{\"index\": \"" + indexName + "\",\"renamed_index\": \"mounted_shared_cache_" + indexName + "\"}"); + mountRequest.addParameter("wait_for_completion", "true"); + mountRequest.addParameter("storage", "shared_cache"); + mountResponse = ObjectPath.createFromResponse(client().performRequest(mountRequest)); + assertNotNull(mountResponse.evaluate("snapshot")); + assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.total")); + assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.successful")); + + // run a search against the index + assertDocs("mounted_shared_cache_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); + } + + private void assertDocs( + String index, + int numDocs, + Set expectedIds, + boolean sourceOnlyRepository, + Version oldVersion, + int numberOfShards + ) throws IOException { + RequestOptions requestOptions = RequestOptions.DEFAULT; + + // run a search against the index + SearchResponse searchResponse = search(index, null, requestOptions); + try { + logger.info(searchResponse); + // check hit count + assertEquals(numDocs, searchResponse.getHits().getTotalHits().value()); + // check that _index is properly set + assertTrue(Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getIndex).allMatch(index::equals)); + // check that all _ids are there + assertEquals(expectedIds, Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toSet())); + // check that _source is present + assertTrue(Arrays.stream(searchResponse.getHits().getHits()).allMatch(SearchHit::hasSource)); + // check that correct _source present for each document + for (SearchHit h : searchResponse.getHits().getHits()) { + assertEquals(sourceForDoc(getIdAsNumeric(h.getId())), h.getSourceAsString()); + } + } finally { + searchResponse.decRef(); + } + + String id = randomFrom(expectedIds); + int num = getIdAsNumeric(id); + // run a search using runtime fields against the index + searchResponse = search( + index, + SearchSourceBuilder.searchSource() + .query(QueryBuilders.matchQuery("val", num)) + .runtimeMappings(Map.of("val", Map.of("type", "long"))), + requestOptions + ); + try { + logger.info(searchResponse); + assertEquals(1, searchResponse.getHits().getTotalHits().value()); + assertEquals(id, searchResponse.getHits().getHits()[0].getId()); + assertEquals(sourceForDoc(num), searchResponse.getHits().getHits()[0].getSourceAsString()); + } finally { + searchResponse.decRef(); + } + + if (sourceOnlyRepository == false) { + // search using reverse sort on val + searchResponse = search( + index, + SearchSourceBuilder.searchSource() + .query(QueryBuilders.matchAllQuery()) + .sort(SortBuilders.fieldSort("val").order(SortOrder.DESC)), + requestOptions + ); + try { + logger.info(searchResponse); + // check sort order + assertEquals( + expectedIds.stream().sorted(Comparator.comparingInt(this::getIdAsNumeric).reversed()).toList(), + Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).toList() + ); + } finally { + searchResponse.decRef(); + } + + // look up postings + searchResponse = search( + index, + SearchSourceBuilder.searchSource().query(QueryBuilders.matchQuery("test", "test" + num)), + requestOptions + ); + try { + logger.info(searchResponse); + // check match + ElasticsearchAssertions.assertSearchHits(searchResponse, id); + } finally { + searchResponse.decRef(); + } + + if (oldVersion.before(Version.fromString("6.0.0"))) { + // search on _type and check that results contain _type information + String randomType = getType(oldVersion, randomFrom(expectedIds)); + long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count(); + searchResponse = search( + index, + SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType)), + requestOptions + ); + try { + logger.info(searchResponse); + assertEquals(typeCount, searchResponse.getHits().getTotalHits().value()); + for (SearchHit hit : searchResponse.getHits().getHits()) { + DocumentField typeField = hit.field("_type"); + assertNotNull(typeField); + assertThat(typeField.getValue(), instanceOf(String.class)); + assertEquals(randomType, typeField.getValue()); + } + } finally { + searchResponse.decRef(); + } + } + + assertThat( + expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/" + index + "/_doc/" + id))) + .getMessage(), + containsString("get operations not allowed on a legacy index") + ); + + // check that shards are skipped based on non-matching date + searchResponse = search( + index, + SearchSourceBuilder.searchSource().query(QueryBuilders.rangeQuery("create_date").from("2020-02-01")), + requestOptions + ); + try { + logger.info(searchResponse); + assertEquals(0, searchResponse.getHits().getTotalHits().value()); + assertEquals(numberOfShards, searchResponse.getSuccessfulShards()); + assertEquals(numberOfShards, searchResponse.getSkippedShards()); + } finally { + searchResponse.decRef(); + } + } + } + + private static SearchResponse search(String index, @Nullable SearchSourceBuilder builder, RequestOptions options) throws IOException { + Request request = new Request("POST", "/" + index + "/_search"); + if (builder != null) { + request.setJsonEntity(builder.toString()); + } + request.setOptions(options); + return SearchResponseUtils.parseSearchResponse(responseAsParser(client().performRequest(request))); + } + + private int getIdAsNumeric(String id) { + return Integer.parseInt(id.substring("testdoc".length())); + } + + private static void closeIndex(RestClient client, String index) throws IOException { + Request request = new Request("POST", "/" + index + "/_close"); + ObjectPath doc = ObjectPath.createFromResponse(client.performRequest(request)); + assertTrue(doc.evaluate("shards_acknowledged")); + } +} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/custom.json b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/custom.json new file mode 100644 index 0000000000000..ae52ccbcce330 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/custom.json @@ -0,0 +1,22 @@ +"_default_": { + "properties": { + "apache2": { + "properties": { + "access": { + "properties": { + "agent": { + "type": "text" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "completion": { + "type": "completion" + } + } +} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/filebeat.json b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/filebeat.json new file mode 100644 index 0000000000000..6fa22f1c36ef9 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/filebeat.json @@ -0,0 +1,681 @@ +"_default_": { + "_meta": { + "version": "5.6.17" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "apache2": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "auditd": { + "properties": { + "log": { + "properties": { + "a0": { + "ignore_above": 1024, + "type": "keyword" + }, + "acct": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "item": { + "ignore_above": 1024, + "type": "keyword" + }, + "items": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ppid": { + "ignore_above": 1024, + "type": "keyword" + }, + "record_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "res": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequence": { + "type": "long" + } + } + } + } + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "fileset": { + "properties": { + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "input_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mysql": { + "properties": { + "error": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "thread_id": { + "type": "long" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "nginx": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "offset": { + "type": "long" + }, + "read_timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "system": { + "properties": { + "auth": { + "properties": { + "groupadd": { + "properties": { + "gid": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssh": { + "properties": { + "dropped_ip": { + "type": "ip" + }, + "event": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sudo": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "pwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + }, + "useradd": { + "properties": { + "gid": { + "type": "long" + }, + "home": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "shell": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "type": "long" + } + } + } + } + }, + "syslog": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } +} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/nested.json b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/nested.json new file mode 100644 index 0000000000000..46dfb5243f06a --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/nested.json @@ -0,0 +1,7 @@ +"_default_": { + "properties": { + "user": { + "type": "nested" + } + } +} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/winlogbeat.json b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/winlogbeat.json new file mode 100644 index 0000000000000..ee4500e10ccfc --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/winlogbeat.json @@ -0,0 +1,1612 @@ +"_default_" : { + "_all" : { + "enabled" : true, + "norms" : false + }, + "dynamic_templates" : [ + { + "message_field" : { + "path_match" : "*message*", + "match_mapping_type" : "string", + "mapping" : { + "fields" : { + "raw" : { + "ignore_above" : 512, + "type" : "keyword" + } + }, + "norms" : false, + "type" : "text" + } + } + }, + { + "tags_field" : { + "path_match" : "*tags*", + "match_mapping_type" : "string", + "mapping" : { + "fields" : { + "raw" : { + "ignore_above" : 512, + "type" : "keyword" + } + }, + "norms" : false, + "type" : "text" + } + } + }, + { + "string_fields" : { + "match" : "*", + "match_mapping_type" : "string", + "mapping" : { + "norms" : false, + "type" : "keyword" + } + } + } + ], + "properties" : { + "@timestamp" : { + "type" : "date", + "include_in_all" : false + }, + "@version" : { + "type" : "keyword", + "include_in_all" : false + }, + "hostIP" : { + "type" : "ip" + }, + "hostIP_geo" : { + "dynamic" : "true", + "properties" : { + "location" : { + "type" : "geo_point" + }, + "postal_code" : { + "type" : "keyword" + } + } + }, + "hostname" : { + "type" : "keyword" + }, + "meta" : { + "include_in_all" : false, + "properties" : { + "processed_at_indexer" : { + "type" : "keyword" + }, + "processed_at_shipper" : { + "type" : "keyword" + }, + "received_from" : { + "type" : "keyword" + } + } + }, + "net" : { + "properties" : { + "dst_ip" : { + "type" : "ip" + }, + "dst_ip_geo" : { + "properties" : { + "location" : { + "type" : "geo_point" + } + } + }, + "dst_nat_ip" : { + "type" : "ip" + }, + "dst_nat_port" : { + "type" : "long" + }, + "dst_port" : { + "type" : "long" + }, + "src_ip" : { + "type" : "ip" + }, + "src_ip_geo" : { + "properties" : { + "location" : { + "type" : "geo_point" + } + } + }, + "src_nat_ip" : { + "type" : "ip" + }, + "src_nat_port" : { + "type" : "long" + }, + "src_port" : { + "type" : "long" + } + } + }, + "type" : { + "type" : "keyword" + } + } +}, +"wineventlog" : { + "_all" : { + "enabled" : true, + "norms" : false + }, + "dynamic_templates" : [ + { + "message_field" : { + "path_match" : "*message*", + "match_mapping_type" : "string", + "mapping" : { + "fields" : { + "raw" : { + "ignore_above" : 512, + "type" : "keyword" + } + }, + "norms" : false, + "type" : "text" + } + } + }, + { + "tags_field" : { + "path_match" : "*tags*", + "match_mapping_type" : "string", + "mapping" : { + "fields" : { + "raw" : { + "ignore_above" : 512, + "type" : "keyword" + } + }, + "norms" : false, + "type" : "text" + } + } + }, + { + "string_fields" : { + "match" : "*", + "match_mapping_type" : "string", + "mapping" : { + "norms" : false, + "type" : "keyword" + } + } + } + ], + "properties" : { + "@timestamp" : { + "type" : "date", + "include_in_all" : false + }, + "@version" : { + "type" : "keyword", + "include_in_all" : false + }, + "beat" : { + "type" : "object" + }, + "eventdata" : { + "properties" : { + "AccessGranted" : { + "type" : "keyword" + }, + "AccessList" : { + "type" : "keyword" + }, + "AccessMask" : { + "type" : "keyword" + }, + "AccessReason" : { + "type" : "keyword" + }, + "AccessRemoved" : { + "type" : "keyword" + }, + "AccountDomain" : { + "type" : "keyword" + }, + "AccountExpires" : { + "type" : "keyword" + }, + "AccountName" : { + "type" : "keyword" + }, + "AccountSessionIdentifier" : { + "type" : "keyword" + }, + "Action" : { + "type" : "keyword" + }, + "ActiveProfile" : { + "type" : "keyword" + }, + "AdditionalInfo" : { + "type" : "keyword" + }, + "AdditionalInfo2" : { + "type" : "keyword" + }, + "AdvancedOptions" : { + "type" : "keyword" + }, + "AhAuthType" : { + "type" : "keyword" + }, + "AlgorithmName" : { + "type" : "keyword" + }, + "AllowedToDelegateTo" : { + "type" : "keyword" + }, + "AppCorrelationID" : { + "type" : "keyword" + }, + "AppInstance" : { + "type" : "keyword" + }, + "AppName" : { + "type" : "keyword" + }, + "Application" : { + "type" : "keyword" + }, + "AttributeLDAPDisplayName" : { + "type" : "keyword" + }, + "AttributeSyntaxOID" : { + "type" : "keyword" + }, + "AttributeValue" : { + "type" : "keyword" + }, + "Attributes" : { + "type" : "keyword" + }, + "AuditPolicyChanges" : { + "type" : "keyword" + }, + "AuditSourceName" : { + "type" : "keyword" + }, + "AuditsDiscarded" : { + "type" : "keyword" + }, + "AuthenticationPackage" : { + "type" : "keyword" + }, + "AuthenticationPackageName" : { + "type" : "keyword" + }, + "AuthenticationProvider" : { + "type" : "keyword" + }, + "AuthenticationServer" : { + "type" : "keyword" + }, + "AuthenticationType" : { + "type" : "keyword" + }, + "BackupPath" : { + "type" : "keyword" + }, + "BackupType" : { + "type" : "keyword" + }, + "CalledStationID" : { + "type" : "keyword" + }, + "CallerProcessId" : { + "type" : "keyword" + }, + "CallerProcessName" : { + "type" : "keyword" + }, + "CallingStationID" : { + "type" : "keyword" + }, + "CalloutId" : { + "type" : "keyword" + }, + "CalloutKey" : { + "type" : "keyword" + }, + "CalloutName" : { + "type" : "keyword" + }, + "CalloutType" : { + "type" : "keyword" + }, + "Categories" : { + "type" : "keyword" + }, + "CategoryId" : { + "type" : "keyword" + }, + "ChangeType" : { + "type" : "keyword" + }, + "Channel" : { + "type" : "keyword" + }, + "CipherType" : { + "type" : "keyword" + }, + "ClientAddress" : { + "type" : "keyword" + }, + "ClientDomain" : { + "type" : "keyword" + }, + "ClientIPAddress" : { + "type" : "keyword" + }, + "ClientLogonId" : { + "type" : "keyword" + }, + "ClientName" : { + "type" : "keyword" + }, + "ClientUserName" : { + "type" : "keyword" + }, + "ComputerAccountChange" : { + "type" : "keyword" + }, + "Conditions" : { + "type" : "keyword" + }, + "ConfigAccessPolicy" : { + "type" : "keyword" + }, + "DCDNSName" : { + "type" : "keyword" + }, + "DHGroup" : { + "type" : "keyword" + }, + "DSName" : { + "type" : "keyword" + }, + "DSType" : { + "type" : "keyword" + }, + "DestAddress" : { + "type" : "keyword" + }, + "DestPort" : { + "type" : "keyword" + }, + "DestinationDRA" : { + "type" : "keyword" + }, + "Direction" : { + "type" : "keyword" + }, + "DisableIntegrityChecks" : { + "type" : "keyword" + }, + "DisabledPrivilegeList" : { + "type" : "keyword" + }, + "DisplayName" : { + "type" : "keyword" + }, + "Disposition" : { + "type" : "keyword" + }, + "DnsHostName" : { + "type" : "keyword" + }, + "DomainBehaviorVersion" : { + "type" : "keyword" + }, + "DomainName" : { + "type" : "keyword" + }, + "DomainPolicyChanged" : { + "type" : "keyword" + }, + "DomainSid" : { + "type" : "keyword" + }, + "Dummy" : { + "type" : "keyword" + }, + "EAPErrorCode" : { + "type" : "keyword" + }, + "EAPReasonCode" : { + "type" : "keyword" + }, + "EAPType" : { + "type" : "keyword" + }, + "ElevatedToken" : { + "type" : "keyword" + }, + "EnabledPrivilegeList" : { + "type" : "keyword" + }, + "EndUSN" : { + "type" : "keyword" + }, + "ErrorCode" : { + "type" : "keyword" + }, + "EspAuthType" : { + "type" : "keyword" + }, + "EventCountTotal" : { + "type" : "keyword" + }, + "EventID" : { + "type" : "keyword" + }, + "EventIdx" : { + "type" : "keyword" + }, + "EventSourceId" : { + "type" : "keyword" + }, + "ExtendedQuarantineState" : { + "type" : "keyword" + }, + "FailureDescription" : { + "type" : "keyword" + }, + "FailureId" : { + "type" : "keyword" + }, + "FailurePoint" : { + "type" : "keyword" + }, + "FailureReason" : { + "type" : "keyword" + }, + "FileName" : { + "type" : "keyword" + }, + "FilterId" : { + "type" : "keyword" + }, + "FilterKey" : { + "type" : "keyword" + }, + "FilterName" : { + "type" : "keyword" + }, + "FilterRTID" : { + "type" : "keyword" + }, + "FilterType" : { + "type" : "keyword" + }, + "FlightSigning" : { + "type" : "keyword" + }, + "ForceLogoff" : { + "type" : "keyword" + }, + "FullyQualifiedSubjectMachineName" : { + "type" : "keyword" + }, + "FullyQualifiedSubjectUserName" : { + "type" : "keyword" + }, + "GPOList" : { + "type" : "keyword" + }, + "Group" : { + "type" : "keyword" + }, + "GroupMembership" : { + "type" : "keyword" + }, + "GroupPolicyApplied" : { + "type" : "keyword" + }, + "HandleId" : { + "type" : "keyword" + }, + "HomeDirectory" : { + "type" : "keyword" + }, + "HomePath" : { + "type" : "keyword" + }, + "HypervisorDebug" : { + "type" : "keyword" + }, + "HypervisorLaunchType" : { + "type" : "keyword" + }, + "HypervisorLoadOptions" : { + "type" : "keyword" + }, + "Identity" : { + "type" : "keyword" + }, + "ImpersonationLevel" : { + "type" : "keyword" + }, + "InboundSpi" : { + "type" : "keyword" + }, + "InitiatorCookie" : { + "type" : "keyword" + }, + "InterfaceName" : { + "type" : "keyword" + }, + "IntfGuid" : { + "type" : "keyword" + }, + "IpAddress" : { + "type" : "keyword" + }, + "IpPort" : { + "type" : "keyword" + }, + "IpProtocol" : { + "type" : "keyword" + }, + "KernelDebug" : { + "type" : "keyword" + }, + "KeyFilePath" : { + "type" : "keyword" + }, + "KeyLength" : { + "type" : "keyword" + }, + "KeyModName" : { + "type" : "keyword" + }, + "KeyName" : { + "type" : "keyword" + }, + "KeyType" : { + "type" : "keyword" + }, + "KeyingModuleName" : { + "type" : "keyword" + }, + "LayerId" : { + "type" : "keyword" + }, + "LayerKey" : { + "type" : "keyword" + }, + "LayerName" : { + "type" : "keyword" + }, + "LayerRTID" : { + "type" : "keyword" + }, + "LifetimeKilobytes" : { + "type" : "keyword" + }, + "LifetimePackets" : { + "type" : "keyword" + }, + "LifetimeSeconds" : { + "type" : "keyword" + }, + "LinkName" : { + "type" : "keyword" + }, + "LmPackageName" : { + "type" : "keyword" + }, + "LoadOptions" : { + "type" : "keyword" + }, + "LocalAddress" : { + "type" : "keyword" + }, + "LocalAddressMask" : { + "type" : "keyword" + }, + "LocalKeyModPort" : { + "type" : "keyword" + }, + "LocalMMPrincipalName" : { + "type" : "keyword" + }, + "LocalMac" : { + "type" : "keyword" + }, + "LocalPort" : { + "type" : "keyword" + }, + "LocalTunnelEndpoint" : { + "type" : "keyword" + }, + "LockoutDuration" : { + "type" : "keyword" + }, + "LockoutObservationWindow" : { + "type" : "keyword" + }, + "LockoutThreshold" : { + "type" : "keyword" + }, + "LogDroppedPacketsEnabled" : { + "type" : "keyword" + }, + "LogSuccessfulConnectionsEnabled" : { + "type" : "keyword" + }, + "LoggingResult" : { + "type" : "keyword" + }, + "LogonGuid" : { + "type" : "keyword" + }, + "LogonHours" : { + "type" : "keyword" + }, + "LogonID" : { + "type" : "keyword" + }, + "LogonProcessName" : { + "type" : "keyword" + }, + "LogonType" : { + "type" : "keyword" + }, + "LogonType_Description" : { + "type" : "keyword" + }, + "MMAuthMethod" : { + "type" : "keyword" + }, + "MMCipherAlg" : { + "type" : "keyword" + }, + "MMFilterID" : { + "type" : "keyword" + }, + "MMImpersonationState" : { + "type" : "keyword" + }, + "MMIntegrityAlg" : { + "type" : "keyword" + }, + "MMLifetime" : { + "type" : "keyword" + }, + "MMSAID" : { + "type" : "keyword" + }, + "MachineAccountQuota" : { + "type" : "keyword" + }, + "MachineInventory" : { + "type" : "keyword" + }, + "MainModeSaId" : { + "type" : "keyword" + }, + "MandatoryLabel" : { + "type" : "keyword" + }, + "MappedName" : { + "type" : "keyword" + }, + "MappingBy" : { + "type" : "keyword" + }, + "MasterKeyId" : { + "type" : "keyword" + }, + "MaxPasswordAge" : { + "type" : "keyword" + }, + "MemberName" : { + "type" : "keyword" + }, + "MemberSid" : { + "type" : "keyword" + }, + "Message" : { + "type" : "keyword" + }, + "MessageID" : { + "type" : "keyword" + }, + "MinPasswordAge" : { + "type" : "keyword" + }, + "MinPasswordLength" : { + "type" : "keyword" + }, + "MixedDomainMode" : { + "type" : "keyword" + }, + "Mode" : { + "type" : "keyword" + }, + "ModifiedObjectProperties" : { + "type" : "keyword" + }, + "Module" : { + "type" : "keyword" + }, + "MulticastFlowsEnabled" : { + "type" : "keyword" + }, + "NASIPv4Address" : { + "type" : "keyword" + }, + "NASIPv6Address" : { + "type" : "keyword" + }, + "NASIdentifier" : { + "type" : "keyword" + }, + "NASPort" : { + "type" : "keyword" + }, + "NASPortType" : { + "type" : "keyword" + }, + "NamingContext" : { + "type" : "keyword" + }, + "NetworkPolicyName" : { + "type" : "keyword" + }, + "NewMaxUsers" : { + "type" : "keyword" + }, + "NewProcessId" : { + "type" : "keyword" + }, + "NewProcessName" : { + "type" : "keyword" + }, + "NewRemark" : { + "type" : "keyword" + }, + "NewSD" : { + "type" : "keyword" + }, + "NewSd" : { + "type" : "keyword" + }, + "NewShareFlags" : { + "type" : "keyword" + }, + "NewState" : { + "type" : "keyword" + }, + "NewTargetUserName" : { + "type" : "keyword" + }, + "NewTime" : { + "type" : "date" + }, + "NewUacValue" : { + "type" : "keyword" + }, + "NewValue" : { + "type" : "keyword" + }, + "NewValueType" : { + "type" : "keyword" + }, + "NotificationPackageName" : { + "type" : "keyword" + }, + "ObjectClass" : { + "type" : "keyword" + }, + "ObjectCollectionName" : { + "type" : "keyword" + }, + "ObjectDN" : { + "type" : "keyword" + }, + "ObjectGUID" : { + "type" : "keyword" + }, + "ObjectIdentifyingProperties" : { + "type" : "keyword" + }, + "ObjectName" : { + "type" : "keyword" + }, + "ObjectProperties" : { + "type" : "keyword" + }, + "ObjectServer" : { + "type" : "keyword" + }, + "ObjectType" : { + "type" : "keyword" + }, + "ObjectValueName" : { + "type" : "keyword" + }, + "OemInformation" : { + "type" : "keyword" + }, + "OldMaxUsers" : { + "type" : "keyword" + }, + "OldRemark" : { + "type" : "keyword" + }, + "OldSD" : { + "type" : "keyword" + }, + "OldSd" : { + "type" : "keyword" + }, + "OldShareFlags" : { + "type" : "keyword" + }, + "OldTargetUserName" : { + "type" : "keyword" + }, + "OldUacValue" : { + "type" : "keyword" + }, + "OldValue" : { + "type" : "keyword" + }, + "OldValueType" : { + "type" : "keyword" + }, + "OpCorrelationID" : { + "type" : "keyword" + }, + "Operation" : { + "type" : "keyword" + }, + "OperationId" : { + "type" : "keyword" + }, + "OperationMode" : { + "type" : "keyword" + }, + "OperationName" : { + "type" : "keyword" + }, + "OperationType" : { + "type" : "keyword" + }, + "Options" : { + "type" : "keyword" + }, + "OutboundSpi" : { + "type" : "keyword" + }, + "PackageName" : { + "type" : "keyword" + }, + "ParentProcessName" : { + "type" : "keyword" + }, + "PasswordHistoryLength" : { + "type" : "keyword" + }, + "PasswordLastSet" : { + "type" : "keyword" + }, + "PasswordProperties" : { + "type" : "keyword" + }, + "PeerMac" : { + "type" : "keyword" + }, + "PeerPrivateAddress" : { + "type" : "keyword" + }, + "Policy" : { + "type" : "keyword" + }, + "PreAuthType" : { + "type" : "keyword" + }, + "PreviousTime" : { + "type" : "date" + }, + "PrimaryGroupId" : { + "type" : "keyword" + }, + "PrivilegeList" : { + "type" : "keyword" + }, + "ProcessID" : { + "type" : "keyword" + }, + "ProcessId" : { + "type" : "keyword" + }, + "ProcessName" : { + "type" : "keyword" + }, + "ProductName" : { + "type" : "keyword" + }, + "Profile" : { + "type" : "keyword" + }, + "ProfileChanged" : { + "type" : "keyword" + }, + "ProfilePath" : { + "type" : "keyword" + }, + "ProfileUsed" : { + "type" : "keyword" + }, + "Profiles" : { + "type" : "keyword" + }, + "Properties" : { + "type" : "keyword" + }, + "Protocol" : { + "type" : "keyword" + }, + "ProviderContextKey" : { + "type" : "keyword" + }, + "ProviderContextName" : { + "type" : "keyword" + }, + "ProviderContextType" : { + "type" : "keyword" + }, + "ProviderKey" : { + "type" : "keyword" + }, + "ProviderName" : { + "type" : "keyword" + }, + "ProviderType" : { + "type" : "keyword" + }, + "ProxyPolicyName" : { + "type" : "keyword" + }, + "PuaCount" : { + "type" : "keyword" + }, + "PuaPolicyId" : { + "type" : "keyword" + }, + "PublisherGuid" : { + "type" : "keyword" + }, + "PublisherID" : { + "type" : "keyword" + }, + "PublisherName" : { + "type" : "keyword" + }, + "QMFilterID" : { + "type" : "keyword" + }, + "QMLimit" : { + "type" : "keyword" + }, + "QuarantineHelpURL" : { + "type" : "keyword" + }, + "QuarantineSessionID" : { + "type" : "keyword" + }, + "QuarantineSessionIdentifier" : { + "type" : "keyword" + }, + "QuarantineState" : { + "type" : "keyword" + }, + "QuarantineSystemHealthResult" : { + "type" : "keyword" + }, + "QuickModeSaId" : { + "type" : "keyword" + }, + "Reason" : { + "type" : "keyword" + }, + "ReasonCode" : { + "type" : "keyword" + }, + "ReasonForRejection" : { + "type" : "keyword" + }, + "ReasonText" : { + "type" : "keyword" + }, + "RecoveryKeyId" : { + "type" : "keyword" + }, + "RecoveryReason" : { + "type" : "keyword" + }, + "RecoveryServer" : { + "type" : "keyword" + }, + "RelativeTargetName" : { + "type" : "keyword" + }, + "RemoteAddress" : { + "type" : "keyword" + }, + "RemoteAddressMask" : { + "type" : "keyword" + }, + "RemoteAdminEnabled" : { + "type" : "keyword" + }, + "RemoteEventLogging" : { + "type" : "keyword" + }, + "RemoteKeyModPort" : { + "type" : "keyword" + }, + "RemoteMMPrincipalName" : { + "type" : "keyword" + }, + "RemoteMachineID" : { + "type" : "keyword" + }, + "RemotePort" : { + "type" : "keyword" + }, + "RemotePrivateAddress" : { + "type" : "keyword" + }, + "RemoteTunnelEndpoint" : { + "type" : "keyword" + }, + "RemoteUserID" : { + "type" : "keyword" + }, + "RequestId" : { + "type" : "keyword" + }, + "RequestType" : { + "type" : "keyword" + }, + "Requester" : { + "type" : "keyword" + }, + "ResourceAttributes" : { + "type" : "keyword" + }, + "ResourceManager" : { + "type" : "keyword" + }, + "ResponderCookie" : { + "type" : "keyword" + }, + "RestrictedAdminMode" : { + "type" : "keyword" + }, + "RestrictedSidCount" : { + "type" : "keyword" + }, + "ReturnCode" : { + "type" : "keyword" + }, + "Role" : { + "type" : "keyword" + }, + "RuleAttr" : { + "type" : "keyword" + }, + "RuleId" : { + "type" : "keyword" + }, + "RuleName" : { + "type" : "keyword" + }, + "SPI" : { + "type" : "keyword" + }, + "SSID" : { + "type" : "keyword" + }, + "SamAccountName" : { + "type" : "keyword" + }, + "ScopeName" : { + "type" : "keyword" + }, + "ScriptPath" : { + "type" : "keyword" + }, + "SecurityDescriptor" : { + "type" : "keyword" + }, + "SecurityPackageName" : { + "type" : "keyword" + }, + "Service" : { + "type" : "keyword" + }, + "ServiceAccount" : { + "type" : "keyword" + }, + "ServiceFileName" : { + "type" : "keyword" + }, + "ServiceName" : { + "type" : "keyword" + }, + "ServicePrincipalNames" : { + "type" : "keyword" + }, + "ServiceSid" : { + "type" : "keyword" + }, + "ServiceStartType" : { + "type" : "keyword" + }, + "ServiceType" : { + "type" : "keyword" + }, + "SessionID" : { + "type" : "keyword" + }, + "SessionId" : { + "type" : "keyword" + }, + "SessionName" : { + "type" : "keyword" + }, + "SettingType" : { + "type" : "keyword" + }, + "SettingValue" : { + "type" : "keyword" + }, + "ShareLocalPath" : { + "type" : "keyword" + }, + "ShareName" : { + "type" : "keyword" + }, + "SidHistory" : { + "type" : "keyword" + }, + "SourceAddr" : { + "type" : "keyword" + }, + "SourceAddress" : { + "type" : "keyword" + }, + "SourceDRA" : { + "type" : "keyword" + }, + "SourceHandleId" : { + "type" : "keyword" + }, + "SourcePort" : { + "type" : "keyword" + }, + "SourceProcessId" : { + "type" : "keyword" + }, + "StartUSN" : { + "type" : "keyword" + }, + "State" : { + "type" : "keyword" + }, + "Status" : { + "type" : "keyword" + }, + "StatusCode" : { + "type" : "keyword" + }, + "StoreUrl" : { + "type" : "keyword" + }, + "SubLayerKey" : { + "type" : "keyword" + }, + "SubLayerName" : { + "type" : "keyword" + }, + "SubLayerType" : { + "type" : "keyword" + }, + "SubStatus" : { + "type" : "keyword" + }, + "SubcategoryGuid" : { + "type" : "keyword" + }, + "SubcategoryId" : { + "type" : "keyword" + }, + "Subject" : { + "type" : "keyword" + }, + "SubjectDomainName" : { + "type" : "keyword" + }, + "SubjectKeyIdentifier" : { + "type" : "keyword" + }, + "SubjectLogonId" : { + "type" : "keyword" + }, + "SubjectMachineName" : { + "type" : "keyword" + }, + "SubjectMachineSID" : { + "type" : "keyword" + }, + "SubjectUserDomainName" : { + "type" : "keyword" + }, + "SubjectUserName" : { + "type" : "keyword" + }, + "SubjectUserSid" : { + "type" : "keyword" + }, + "TargetDomainName" : { + "type" : "keyword" + }, + "TargetHandleId" : { + "type" : "keyword" + }, + "TargetInfo" : { + "type" : "keyword" + }, + "TargetLinkedLogonId" : { + "type" : "keyword" + }, + "TargetLogonGuid" : { + "type" : "keyword" + }, + "TargetLogonId" : { + "type" : "keyword" + }, + "TargetOutboundDomainName" : { + "type" : "keyword" + }, + "TargetOutboundUserName" : { + "type" : "keyword" + }, + "TargetProcessId" : { + "type" : "keyword" + }, + "TargetServerName" : { + "type" : "keyword" + }, + "TargetSid" : { + "type" : "keyword" + }, + "TargetUserName" : { + "type" : "keyword" + }, + "TargetUserSid" : { + "type" : "keyword" + }, + "TaskContent" : { + "type" : "keyword" + }, + "TaskContentNew" : { + "type" : "keyword" + }, + "TaskName" : { + "type" : "keyword" + }, + "TemplateContent" : { + "type" : "keyword" + }, + "TemplateDSObjectFQDN" : { + "type" : "keyword" + }, + "TemplateInternalName" : { + "type" : "keyword" + }, + "TemplateOID" : { + "type" : "keyword" + }, + "TemplateSchemaVersion" : { + "type" : "keyword" + }, + "TemplateVersion" : { + "type" : "keyword" + }, + "TestSigning" : { + "type" : "keyword" + }, + "TicketEncryptionType" : { + "type" : "keyword" + }, + "TicketOptions" : { + "type" : "keyword" + }, + "TokenElevationType" : { + "type" : "keyword" + }, + "TrafficSelectorId" : { + "type" : "keyword" + }, + "TransactionId" : { + "type" : "keyword" + }, + "TransmittedServices" : { + "type" : "keyword" + }, + "TransportFilterId" : { + "type" : "keyword" + }, + "TreeDelete" : { + "type" : "keyword" + }, + "TunnelId" : { + "type" : "keyword" + }, + "UserAccountControl" : { + "type" : "keyword" + }, + "UserName" : { + "type" : "keyword" + }, + "UserParameters" : { + "type" : "keyword" + }, + "UserPrincipalName" : { + "type" : "keyword" + }, + "UserSid" : { + "type" : "keyword" + }, + "UserWorkstations" : { + "type" : "keyword" + }, + "VirtualAccount" : { + "type" : "keyword" + }, + "VsmLaunchType" : { + "type" : "keyword" + }, + "Weight" : { + "type" : "keyword" + }, + "Workstation" : { + "type" : "keyword" + }, + "WorkstationName" : { + "type" : "keyword" + }, + "param1" : { + "type" : "keyword" + }, + "param2" : { + "type" : "keyword" + }, + "param3" : { + "type" : "keyword" + }, + "param4" : { + "type" : "keyword" + }, + "param5" : { + "type" : "keyword" + }, + "param6" : { + "type" : "keyword" + }, + "param7" : { + "type" : "keyword" + }, + "param8" : { + "type" : "keyword" + }, + "param9" : { + "type" : "keyword" + }, + "xml_name" : { + "type" : "keyword" + } + } + }, + "eventlog" : { + "properties" : { + "computer" : { + "type" : "keyword" + }, + "eventID" : { + "type" : "long" + }, + "event_recordID" : { + "type" : "long" + }, + "keywords" : { + "type" : "keyword" + }, + "log_name" : { + "type" : "keyword" + }, + "processID" : { + "type" : "long" + }, + "source" : { + "type" : "keyword" + }, + "task_category" : { + "type" : "keyword" + }, + "threadID" : { + "type" : "long" + } + } + }, + "hostIP" : { + "type" : "ip" + }, + "hostIP_geo" : { + "dynamic" : "true", + "properties" : { + "location" : { + "type" : "geo_point" + }, + "postal_code" : { + "type" : "keyword" + } + } + }, + "hostname" : { + "type" : "keyword" + }, + "message" : { + "type" : "text", + "norms" : false, + "fields" : { + "raw" : { + "type" : "keyword", + "ignore_above" : 512 + } + } + }, + "message_error" : { + "type" : "text", + "norms" : false, + "fields" : { + "raw" : { + "type" : "keyword", + "ignore_above" : 512 + } + } + }, + "meta" : { + "include_in_all" : false, + "properties" : { + "beat_type" : { + "type" : "keyword" + }, + "beat_version" : { + "type" : "keyword" + }, + "diff_indexer_shipper_ms" : { + "type" : "long" + }, + "diff_shipper_timestamp_ms" : { + "type" : "long" + }, + "kafka_topic" : { + "type" : "keyword" + }, + "processed_at_indexer" : { + "type" : "keyword" + }, + "processed_at_shipper" : { + "type" : "keyword" + }, + "received_at_indexer" : { + "type" : "date" + }, + "received_at_shipper" : { + "type" : "date" + }, + "received_from" : { + "type" : "keyword" + } + } + }, + "net" : { + "properties" : { + "dst_ip" : { + "type" : "ip" + }, + "dst_ip_geo" : { + "properties" : { + "location" : { + "type" : "geo_point" + } + } + }, + "dst_nat_ip" : { + "type" : "ip" + }, + "dst_nat_port" : { + "type" : "long" + }, + "dst_port" : { + "type" : "long" + }, + "src_domain" : { + "type" : "keyword" + }, + "src_ip" : { + "type" : "ip" + }, + "src_ip_geo" : { + "properties" : { + "location" : { + "type" : "geo_point" + } + } + }, + "src_nat_ip" : { + "type" : "ip" + }, + "src_nat_port" : { + "type" : "long" + }, + "src_port" : { + "type" : "long" + }, + "src_user" : { + "type" : "keyword" + }, + "src_user_department" : { + "type" : "keyword" + }, + "src_user_department_number" : { + "type" : "keyword" + }, + "src_user_display_name" : { + "type" : "keyword" + }, + "src_user_title" : { + "type" : "keyword" + } + } + }, + "severity" : { + "type" : "keyword" + }, + "tags" : { + "type" : "text", + "norms" : false, + "fields" : { + "raw" : { + "type" : "keyword", + "ignore_above" : 512 + } + } + }, + "type" : { + "type" : "keyword" + }, + "user" : { + "type" : "object" + } + } +} From 1bd696d0162896dc0f0f78cbf69afdad2c9e17ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 22 Nov 2024 09:53:41 +0100 Subject: [PATCH 03/24] iter --- .../repository-old-versions-7x/build.gradle | 139 +- .../oldrepos7x/OldMappingsIT.java | 194 ++ .../org/elasticsearch/oldrepos7x/custom.json | 21 + .../elasticsearch/oldrepos7x/filebeat.json | 679 +++++++ .../org/elasticsearch/oldrepos7x/nested.json | 5 + .../elasticsearch/oldrepos7x/winlogbeat.json | 1610 ++++++++++++++++ .../oldrepos/DocValueOnlyFieldsIT.java | 202 --- .../elasticsearch/oldrepos/OldMappingsIT.java | 361 ---- .../oldrepos/OldRepositoryAccessIT.java | 512 ------ .../org/elasticsearch/oldrepos/custom.json | 22 - .../org/elasticsearch/oldrepos/filebeat.json | 681 ------- .../org/elasticsearch/oldrepos/nested.json | 7 - .../elasticsearch/oldrepos/winlogbeat.json | 1612 ----------------- 13 files changed, 2514 insertions(+), 3531 deletions(-) create mode 100644 x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java create mode 100644 x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/custom.json create mode 100644 x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/filebeat.json create mode 100644 x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/nested.json create mode 100644 x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/winlogbeat.json delete mode 100644 x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java delete mode 100644 x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java delete mode 100644 x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java delete mode 100644 x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/custom.json delete mode 100644 x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/filebeat.json delete mode 100644 x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/nested.json delete mode 100644 x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/winlogbeat.json diff --git a/x-pack/qa/repository-old-versions-7x/build.gradle b/x-pack/qa/repository-old-versions-7x/build.gradle index 1c0ce552795d1..2dcbd610cfe78 100644 --- a/x-pack/qa/repository-old-versions-7x/build.gradle +++ b/x-pack/qa/repository-old-versions-7x/build.gradle @@ -1,138 +1,9 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import org.elasticsearch.gradle.Architecture -import org.elasticsearch.gradle.OS +import org.elasticsearch.gradle.internal.test.RestIntegTestTask import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.internal.BwcVersions -import org.elasticsearch.gradle.internal.info.BuildParams -import org.elasticsearch.gradle.internal.test.AntFixture -import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -import org.elasticsearch.gradle.transform.UnzipTransform -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE -apply plugin: 'elasticsearch.jdk-download' -apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.internal-java-rest-test' -apply plugin: 'elasticsearch.internal-yaml-rest-test' -apply plugin: 'elasticsearch.rest-resources' - -jdks { - legacy { - vendor = 'adoptium' - version = '8u302+b08' - platform = OS.current().name().toLowerCase() - architecture = Architecture.current().name().toLowerCase() - } -} - -restResources { - restApi { - include '_common', 'search' - } - restTests { - includeCore 'search/390_doc_values_search.yml' - } -} - -if (OS.current() == OS.MAC && Architecture.current() == Architecture.AARCH64) { - jdks.legacy.vendor = 'zulu' - jdks.legacy.distributionVersion = '8.56.0.23' -} - -if (OS.current() == OS.WINDOWS) { - logger.warn("Disabling repository-old-versions tests because we can't get the pid file on windows") -} else { - int currentMajorVersion = org.elasticsearch.gradle.VersionProperties.elasticsearchVersion.major - for (String versionString : ['7.17.25']) { - Version version = Version.fromString(versionString) - String versionNoDots = version.toString().replace('.', '_') -// String configName = "es${versionNoDots}" - -// def config = configurations.create(configName) -// config.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); -// dependencies.add(configName, artifact) - - String repoLocation = "${buildDir}/cluster/shared/repo/${versionNoDots}" - String clusterName = versionNoDots - - def testClusterProvider = testClusters.register(clusterName) { - testDistribution = 'DEFAULT' - numberOfNodes = 2 - versions = [project.version, project.version] // to test full cluster restart - - setting 'path.repo', repoLocation, IGNORE_VALUE - setting 'xpack.license.self_generated.type', 'trial' - - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - - setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' - setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' - } - - def oldCluster = testClusters.register("oldES${versionNoDots}Cluster") { - testDistribution = 'DEFAULT' - numberOfNodes = 1 - versions = [versionString] - setting 'path.repo', repoLocation, IGNORE_VALUE - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' - setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' - } - - tasks.register("javaRestTestBeforeRestart#${versionNoDots}", StandaloneRestIntegTestTask) { - useCluster testClusterProvider - useCluster testClusters.named("oldES${versionNoDots}Cluster") - doFirst { - delete(repoLocation) - mkdir(repoLocation) - } - systemProperty 'tests.after_restart', 'false' - } - - tasks.register("javaRestTestAfterRestart#${versionNoDots}", StandaloneRestIntegTestTask) { - useCluster testClusterProvider - dependsOn "javaRestTestBeforeRestart#${versionNoDots}" - useCluster testClusters.named("oldES${versionNoDots}Cluster") - systemProperty 'tests.after_restart', 'true' - - doFirst { - testClusterProvider.get().goToNextVersion() - } - } - - tasks.matching { it.name.startsWith("javaRestTest") && it.name.endsWith(versionNoDots) }.configureEach { - doFirst { - def oldClusterPortProvider = testClusters.named("oldES${versionNoDots}Cluster").map { cluster -> - def local = cluster.allHttpSocketURI.find { it.startsWith("127.0.0.1")} - return local.substring(local.lastIndexOf(':') + 1) - } - it.nonInputProperties.systemProperty "tests.es.port", "${-> oldClusterPortProvider.get()}" - println "oldClusterPortProvider " + oldClusterPortProvider.get() - println "allRemoteAccessPortURI " + testClusters.named("oldES${versionNoDots}Cluster").get().allRemoteAccessPortURI - println "allHttpSocketURI " + testClusters.named("oldES${versionNoDots}Cluster").get().allHttpSocketURI - println "httpSocketURI " + testClusters.named("oldES${versionNoDots}Cluster").get().httpSocketURI - } - it.nonInputProperties.systemProperty "tests.repo.location", repoLocation - it.systemProperty "tests.es.version", version.toString() - - /* Use a closure on the string to delay evaluation until right before we - * run the integration tests so that we can be sure that the file is - * ready. */ - it.nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusterProvider.get().allHttpSocketURI.join(",")}") - it.nonInputProperties.systemProperty('tests.clustername', "${-> testClusterProvider.get().getName()}") - } - - tasks.named("check").configure { - dependsOn "javaRestTestAfterRestart#${versionNoDots}" - } - } -} +tasks.named("javaRestTest").configure { + usesDefaultDistribution() + usesBwcDistribution(Version.fromString("7.17.25")) +} \ No newline at end of file diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java new file mode 100644 index 0000000000000..e50e9ce590c86 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos7x; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.Version; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class OldMappingsIT extends ESRestTestCase { + + public static TemporaryFolder repoDirectory = new TemporaryFolder(); + + public static ElasticsearchCluster currentCluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .distribution(DistributionType.DEFAULT) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.ml.enabled", "false") + .setting("path.repo", () -> repoDirectory.getRoot().getPath()) + .build(); + + public static ElasticsearchCluster oldCluster = ElasticsearchCluster.local() + .version(Version.fromString("7.17.25")) + .distribution(DistributionType.DEFAULT) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.ml.enabled", "false") + .setting("path.repo", () -> repoDirectory.getRoot().getPath()) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(oldCluster).around(currentCluster); + + @Override + protected String getTestRestCluster() { + return oldCluster.getHttpAddresses(); + } + + private Request createIndex(String indexName, String file) throws IOException { + Request createIndex = new Request("PUT", "/" + indexName); + int numberOfShards = randomIntBetween(1, 3); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("settings") + .field("index.number_of_shards", numberOfShards) + .endObject() + .startObject("mappings"); + builder.rawValue(getClass().getResourceAsStream(file), XContentType.JSON); + builder.endObject().endObject(); + + createIndex.setJsonEntity(Strings.toString(builder)); + return createIndex; + } + + @Before + public void setupIndex() throws IOException { + // String repoLocation = PathUtils.get(System.getProperty("tests.repo.location")) + // .resolve(RandomizedTest.getContext().getTargetClass().getName()) + // .toString(); + + String repoLocation = repoDirectory.getRoot().getPath(); + + String repoName = "old_mappings_repo"; + String snapshotName = "snap"; + List indices; + indices = Arrays.asList("filebeat", "custom", "nested"); + + List oldClusterHosts = parseClusterHosts(oldCluster.getHttpAddresses()); + List clusterHosts = parseClusterHosts(currentCluster.getHttpAddresses()); + // try (RestClient oldEsClient = client()) { + try ( + RestClient oldEsClient = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build(); + RestClient esClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()])).build(); + ) { + assertOK(oldEsClient.performRequest(createIndex("filebeat", "filebeat.json"))); + + assertOK(oldEsClient.performRequest(createIndex("custom", "custom.json"))); + assertOK(oldEsClient.performRequest(createIndex("nested", "nested.json"))); + + Request doc1 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "1"); + doc1.addParameter("refresh", "true"); + XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() + .startObject() + .startObject("apache2") + .startObject("access") + .field("url", "myurl1") + .field("agent", "agent1") + .endObject() + .endObject() + .endObject(); + doc1.setJsonEntity(Strings.toString(bodyDoc1)); + assertOK(oldEsClient.performRequest(doc1)); + + Request doc2 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "2"); + doc2.addParameter("refresh", "true"); + XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() + .startObject() + .startObject("apache2") + .startObject("access") + .field("url", "myurl2") + .field("agent", "agent2 agent2") + .endObject() + .endObject() + .field("completion", "some_value") + .endObject(); + doc2.setJsonEntity(Strings.toString(bodyDoc2)); + assertOK(oldEsClient.performRequest(doc2)); + + Request doc3 = new Request("PUT", "/" + "nested" + "/" + "_doc" + "/" + "1"); + doc3.addParameter("refresh", "true"); + XContentBuilder bodyDoc3 = XContentFactory.jsonBuilder() + .startObject() + .field("group", "fans") + .startArray("user") + .startObject() + .field("first", "John") + .field("last", "Smith") + .endObject() + .startObject() + .field("first", "Alice") + .field("last", "White") + .endObject() + .endArray() + .endObject(); + doc3.setJsonEntity(Strings.toString(bodyDoc3)); + assertOK(oldEsClient.performRequest(doc3)); + + Request getSettingsRequest = new Request("GET", "/_cluster/settings?include_defaults=true"); + Map response = entityAsMap(oldEsClient.performRequest(getSettingsRequest)); + assertEquals(repoLocation, ((List) (XContentMapValues.extractValue("defaults.path.repo", response))).get(0)); + + // register repo on old ES and take snapshot + Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(oldEsClient.performRequest(createRepoRequest)); + + Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); + assertOK(oldEsClient.performRequest(createSnapshotRequest)); + // } + + // register repo on new ES and restore snapshot + Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest2.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(esClient.performRequest(createRepoRequest2)); + + final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); + createRestoreRequest.addParameter("wait_for_completion", "true"); + createRestoreRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); + createRestoreRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + assertOK(esClient.performRequest(createRestoreRequest)); + } + } + + public void testMappingOk() throws IOException { + // Request mappingRequest = new Request("GET", "/" + "filebeat" + "/_mapping"); + // Map mapping = entityAsMap(client().performRequest(mappingRequest)); + // assertNotNull(XContentMapValues.extractValue(mapping, "filebeat", "mappings", "properties", "apache2")); + } + +} diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/custom.json b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/custom.json new file mode 100644 index 0000000000000..b5865080d9abf --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/custom.json @@ -0,0 +1,21 @@ +"properties": { + "apache2": { + "properties": { + "access": { + "properties": { + "agent": { + "type": "text" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "completion": { + "type": "completion" + } +} + diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/filebeat.json b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/filebeat.json new file mode 100644 index 0000000000000..ff2a12c7a44a0 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/filebeat.json @@ -0,0 +1,679 @@ +"_meta": { + "version": "5.6.17" +}, +"date_detection": false, +"dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } +], +"properties": { + "@timestamp": { + "type": "date" + }, + "apache2": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "auditd": { + "properties": { + "log": { + "properties": { + "a0": { + "ignore_above": 1024, + "type": "keyword" + }, + "acct": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "item": { + "ignore_above": 1024, + "type": "keyword" + }, + "items": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ppid": { + "ignore_above": 1024, + "type": "keyword" + }, + "record_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "res": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequence": { + "type": "long" + } + } + } + } + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "fileset": { + "properties": { + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "input_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mysql": { + "properties": { + "error": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "thread_id": { + "type": "long" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "nginx": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "offset": { + "type": "long" + }, + "read_timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "system": { + "properties": { + "auth": { + "properties": { + "groupadd": { + "properties": { + "gid": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssh": { + "properties": { + "dropped_ip": { + "type": "ip" + }, + "event": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sudo": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "pwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + }, + "useradd": { + "properties": { + "gid": { + "type": "long" + }, + "home": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "shell": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "type": "long" + } + } + } + } + }, + "syslog": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } +} diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/nested.json b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/nested.json new file mode 100644 index 0000000000000..e8fd42f1d7c58 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/nested.json @@ -0,0 +1,5 @@ +"properties": { + "user": { + "type": "nested" + } +} diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/winlogbeat.json b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/winlogbeat.json new file mode 100644 index 0000000000000..8fcd4048bf113 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/winlogbeat.json @@ -0,0 +1,1610 @@ +"_all" : { + "enabled" : true, + "norms" : false +}, +"dynamic_templates" : [ + { + "message_field" : { + "path_match" : "*message*", + "match_mapping_type" : "string", + "mapping" : { + "fields" : { + "raw" : { + "ignore_above" : 512, + "type" : "keyword" + } + }, + "norms" : false, + "type" : "text" + } + } + }, + { + "tags_field" : { + "path_match" : "*tags*", + "match_mapping_type" : "string", + "mapping" : { + "fields" : { + "raw" : { + "ignore_above" : 512, + "type" : "keyword" + } + }, + "norms" : false, + "type" : "text" + } + } + }, + { + "string_fields" : { + "match" : "*", + "match_mapping_type" : "string", + "mapping" : { + "norms" : false, + "type" : "keyword" + } + } + } +], +"properties" : { + "@timestamp" : { + "type" : "date", + "include_in_all" : false + }, + "@version" : { + "type" : "keyword", + "include_in_all" : false + }, + "hostIP" : { + "type" : "ip" + }, + "hostIP_geo" : { + "dynamic" : "true", + "properties" : { + "location" : { + "type" : "geo_point" + }, + "postal_code" : { + "type" : "keyword" + } + } + }, + "hostname" : { + "type" : "keyword" + }, + "meta" : { + "include_in_all" : false, + "properties" : { + "processed_at_indexer" : { + "type" : "keyword" + }, + "processed_at_shipper" : { + "type" : "keyword" + }, + "received_from" : { + "type" : "keyword" + } + } + }, + "net" : { + "properties" : { + "dst_ip" : { + "type" : "ip" + }, + "dst_ip_geo" : { + "properties" : { + "location" : { + "type" : "geo_point" + } + } + }, + "dst_nat_ip" : { + "type" : "ip" + }, + "dst_nat_port" : { + "type" : "long" + }, + "dst_port" : { + "type" : "long" + }, + "src_ip" : { + "type" : "ip" + }, + "src_ip_geo" : { + "properties" : { + "location" : { + "type" : "geo_point" + } + } + }, + "src_nat_ip" : { + "type" : "ip" + }, + "src_nat_port" : { + "type" : "long" + }, + "src_port" : { + "type" : "long" + } + } + }, + "type" : { + "type" : "keyword" + } +} +}, +"wineventlog" : { +"_all" : { + "enabled" : true, + "norms" : false +}, +"dynamic_templates" : [ + { + "message_field" : { + "path_match" : "*message*", + "match_mapping_type" : "string", + "mapping" : { + "fields" : { + "raw" : { + "ignore_above" : 512, + "type" : "keyword" + } + }, + "norms" : false, + "type" : "text" + } + } + }, + { + "tags_field" : { + "path_match" : "*tags*", + "match_mapping_type" : "string", + "mapping" : { + "fields" : { + "raw" : { + "ignore_above" : 512, + "type" : "keyword" + } + }, + "norms" : false, + "type" : "text" + } + } + }, + { + "string_fields" : { + "match" : "*", + "match_mapping_type" : "string", + "mapping" : { + "norms" : false, + "type" : "keyword" + } + } + } +], +"properties" : { + "@timestamp" : { + "type" : "date", + "include_in_all" : false + }, + "@version" : { + "type" : "keyword", + "include_in_all" : false + }, + "beat" : { + "type" : "object" + }, + "eventdata" : { + "properties" : { + "AccessGranted" : { + "type" : "keyword" + }, + "AccessList" : { + "type" : "keyword" + }, + "AccessMask" : { + "type" : "keyword" + }, + "AccessReason" : { + "type" : "keyword" + }, + "AccessRemoved" : { + "type" : "keyword" + }, + "AccountDomain" : { + "type" : "keyword" + }, + "AccountExpires" : { + "type" : "keyword" + }, + "AccountName" : { + "type" : "keyword" + }, + "AccountSessionIdentifier" : { + "type" : "keyword" + }, + "Action" : { + "type" : "keyword" + }, + "ActiveProfile" : { + "type" : "keyword" + }, + "AdditionalInfo" : { + "type" : "keyword" + }, + "AdditionalInfo2" : { + "type" : "keyword" + }, + "AdvancedOptions" : { + "type" : "keyword" + }, + "AhAuthType" : { + "type" : "keyword" + }, + "AlgorithmName" : { + "type" : "keyword" + }, + "AllowedToDelegateTo" : { + "type" : "keyword" + }, + "AppCorrelationID" : { + "type" : "keyword" + }, + "AppInstance" : { + "type" : "keyword" + }, + "AppName" : { + "type" : "keyword" + }, + "Application" : { + "type" : "keyword" + }, + "AttributeLDAPDisplayName" : { + "type" : "keyword" + }, + "AttributeSyntaxOID" : { + "type" : "keyword" + }, + "AttributeValue" : { + "type" : "keyword" + }, + "Attributes" : { + "type" : "keyword" + }, + "AuditPolicyChanges" : { + "type" : "keyword" + }, + "AuditSourceName" : { + "type" : "keyword" + }, + "AuditsDiscarded" : { + "type" : "keyword" + }, + "AuthenticationPackage" : { + "type" : "keyword" + }, + "AuthenticationPackageName" : { + "type" : "keyword" + }, + "AuthenticationProvider" : { + "type" : "keyword" + }, + "AuthenticationServer" : { + "type" : "keyword" + }, + "AuthenticationType" : { + "type" : "keyword" + }, + "BackupPath" : { + "type" : "keyword" + }, + "BackupType" : { + "type" : "keyword" + }, + "CalledStationID" : { + "type" : "keyword" + }, + "CallerProcessId" : { + "type" : "keyword" + }, + "CallerProcessName" : { + "type" : "keyword" + }, + "CallingStationID" : { + "type" : "keyword" + }, + "CalloutId" : { + "type" : "keyword" + }, + "CalloutKey" : { + "type" : "keyword" + }, + "CalloutName" : { + "type" : "keyword" + }, + "CalloutType" : { + "type" : "keyword" + }, + "Categories" : { + "type" : "keyword" + }, + "CategoryId" : { + "type" : "keyword" + }, + "ChangeType" : { + "type" : "keyword" + }, + "Channel" : { + "type" : "keyword" + }, + "CipherType" : { + "type" : "keyword" + }, + "ClientAddress" : { + "type" : "keyword" + }, + "ClientDomain" : { + "type" : "keyword" + }, + "ClientIPAddress" : { + "type" : "keyword" + }, + "ClientLogonId" : { + "type" : "keyword" + }, + "ClientName" : { + "type" : "keyword" + }, + "ClientUserName" : { + "type" : "keyword" + }, + "ComputerAccountChange" : { + "type" : "keyword" + }, + "Conditions" : { + "type" : "keyword" + }, + "ConfigAccessPolicy" : { + "type" : "keyword" + }, + "DCDNSName" : { + "type" : "keyword" + }, + "DHGroup" : { + "type" : "keyword" + }, + "DSName" : { + "type" : "keyword" + }, + "DSType" : { + "type" : "keyword" + }, + "DestAddress" : { + "type" : "keyword" + }, + "DestPort" : { + "type" : "keyword" + }, + "DestinationDRA" : { + "type" : "keyword" + }, + "Direction" : { + "type" : "keyword" + }, + "DisableIntegrityChecks" : { + "type" : "keyword" + }, + "DisabledPrivilegeList" : { + "type" : "keyword" + }, + "DisplayName" : { + "type" : "keyword" + }, + "Disposition" : { + "type" : "keyword" + }, + "DnsHostName" : { + "type" : "keyword" + }, + "DomainBehaviorVersion" : { + "type" : "keyword" + }, + "DomainName" : { + "type" : "keyword" + }, + "DomainPolicyChanged" : { + "type" : "keyword" + }, + "DomainSid" : { + "type" : "keyword" + }, + "Dummy" : { + "type" : "keyword" + }, + "EAPErrorCode" : { + "type" : "keyword" + }, + "EAPReasonCode" : { + "type" : "keyword" + }, + "EAPType" : { + "type" : "keyword" + }, + "ElevatedToken" : { + "type" : "keyword" + }, + "EnabledPrivilegeList" : { + "type" : "keyword" + }, + "EndUSN" : { + "type" : "keyword" + }, + "ErrorCode" : { + "type" : "keyword" + }, + "EspAuthType" : { + "type" : "keyword" + }, + "EventCountTotal" : { + "type" : "keyword" + }, + "EventID" : { + "type" : "keyword" + }, + "EventIdx" : { + "type" : "keyword" + }, + "EventSourceId" : { + "type" : "keyword" + }, + "ExtendedQuarantineState" : { + "type" : "keyword" + }, + "FailureDescription" : { + "type" : "keyword" + }, + "FailureId" : { + "type" : "keyword" + }, + "FailurePoint" : { + "type" : "keyword" + }, + "FailureReason" : { + "type" : "keyword" + }, + "FileName" : { + "type" : "keyword" + }, + "FilterId" : { + "type" : "keyword" + }, + "FilterKey" : { + "type" : "keyword" + }, + "FilterName" : { + "type" : "keyword" + }, + "FilterRTID" : { + "type" : "keyword" + }, + "FilterType" : { + "type" : "keyword" + }, + "FlightSigning" : { + "type" : "keyword" + }, + "ForceLogoff" : { + "type" : "keyword" + }, + "FullyQualifiedSubjectMachineName" : { + "type" : "keyword" + }, + "FullyQualifiedSubjectUserName" : { + "type" : "keyword" + }, + "GPOList" : { + "type" : "keyword" + }, + "Group" : { + "type" : "keyword" + }, + "GroupMembership" : { + "type" : "keyword" + }, + "GroupPolicyApplied" : { + "type" : "keyword" + }, + "HandleId" : { + "type" : "keyword" + }, + "HomeDirectory" : { + "type" : "keyword" + }, + "HomePath" : { + "type" : "keyword" + }, + "HypervisorDebug" : { + "type" : "keyword" + }, + "HypervisorLaunchType" : { + "type" : "keyword" + }, + "HypervisorLoadOptions" : { + "type" : "keyword" + }, + "Identity" : { + "type" : "keyword" + }, + "ImpersonationLevel" : { + "type" : "keyword" + }, + "InboundSpi" : { + "type" : "keyword" + }, + "InitiatorCookie" : { + "type" : "keyword" + }, + "InterfaceName" : { + "type" : "keyword" + }, + "IntfGuid" : { + "type" : "keyword" + }, + "IpAddress" : { + "type" : "keyword" + }, + "IpPort" : { + "type" : "keyword" + }, + "IpProtocol" : { + "type" : "keyword" + }, + "KernelDebug" : { + "type" : "keyword" + }, + "KeyFilePath" : { + "type" : "keyword" + }, + "KeyLength" : { + "type" : "keyword" + }, + "KeyModName" : { + "type" : "keyword" + }, + "KeyName" : { + "type" : "keyword" + }, + "KeyType" : { + "type" : "keyword" + }, + "KeyingModuleName" : { + "type" : "keyword" + }, + "LayerId" : { + "type" : "keyword" + }, + "LayerKey" : { + "type" : "keyword" + }, + "LayerName" : { + "type" : "keyword" + }, + "LayerRTID" : { + "type" : "keyword" + }, + "LifetimeKilobytes" : { + "type" : "keyword" + }, + "LifetimePackets" : { + "type" : "keyword" + }, + "LifetimeSeconds" : { + "type" : "keyword" + }, + "LinkName" : { + "type" : "keyword" + }, + "LmPackageName" : { + "type" : "keyword" + }, + "LoadOptions" : { + "type" : "keyword" + }, + "LocalAddress" : { + "type" : "keyword" + }, + "LocalAddressMask" : { + "type" : "keyword" + }, + "LocalKeyModPort" : { + "type" : "keyword" + }, + "LocalMMPrincipalName" : { + "type" : "keyword" + }, + "LocalMac" : { + "type" : "keyword" + }, + "LocalPort" : { + "type" : "keyword" + }, + "LocalTunnelEndpoint" : { + "type" : "keyword" + }, + "LockoutDuration" : { + "type" : "keyword" + }, + "LockoutObservationWindow" : { + "type" : "keyword" + }, + "LockoutThreshold" : { + "type" : "keyword" + }, + "LogDroppedPacketsEnabled" : { + "type" : "keyword" + }, + "LogSuccessfulConnectionsEnabled" : { + "type" : "keyword" + }, + "LoggingResult" : { + "type" : "keyword" + }, + "LogonGuid" : { + "type" : "keyword" + }, + "LogonHours" : { + "type" : "keyword" + }, + "LogonID" : { + "type" : "keyword" + }, + "LogonProcessName" : { + "type" : "keyword" + }, + "LogonType" : { + "type" : "keyword" + }, + "LogonType_Description" : { + "type" : "keyword" + }, + "MMAuthMethod" : { + "type" : "keyword" + }, + "MMCipherAlg" : { + "type" : "keyword" + }, + "MMFilterID" : { + "type" : "keyword" + }, + "MMImpersonationState" : { + "type" : "keyword" + }, + "MMIntegrityAlg" : { + "type" : "keyword" + }, + "MMLifetime" : { + "type" : "keyword" + }, + "MMSAID" : { + "type" : "keyword" + }, + "MachineAccountQuota" : { + "type" : "keyword" + }, + "MachineInventory" : { + "type" : "keyword" + }, + "MainModeSaId" : { + "type" : "keyword" + }, + "MandatoryLabel" : { + "type" : "keyword" + }, + "MappedName" : { + "type" : "keyword" + }, + "MappingBy" : { + "type" : "keyword" + }, + "MasterKeyId" : { + "type" : "keyword" + }, + "MaxPasswordAge" : { + "type" : "keyword" + }, + "MemberName" : { + "type" : "keyword" + }, + "MemberSid" : { + "type" : "keyword" + }, + "Message" : { + "type" : "keyword" + }, + "MessageID" : { + "type" : "keyword" + }, + "MinPasswordAge" : { + "type" : "keyword" + }, + "MinPasswordLength" : { + "type" : "keyword" + }, + "MixedDomainMode" : { + "type" : "keyword" + }, + "Mode" : { + "type" : "keyword" + }, + "ModifiedObjectProperties" : { + "type" : "keyword" + }, + "Module" : { + "type" : "keyword" + }, + "MulticastFlowsEnabled" : { + "type" : "keyword" + }, + "NASIPv4Address" : { + "type" : "keyword" + }, + "NASIPv6Address" : { + "type" : "keyword" + }, + "NASIdentifier" : { + "type" : "keyword" + }, + "NASPort" : { + "type" : "keyword" + }, + "NASPortType" : { + "type" : "keyword" + }, + "NamingContext" : { + "type" : "keyword" + }, + "NetworkPolicyName" : { + "type" : "keyword" + }, + "NewMaxUsers" : { + "type" : "keyword" + }, + "NewProcessId" : { + "type" : "keyword" + }, + "NewProcessName" : { + "type" : "keyword" + }, + "NewRemark" : { + "type" : "keyword" + }, + "NewSD" : { + "type" : "keyword" + }, + "NewSd" : { + "type" : "keyword" + }, + "NewShareFlags" : { + "type" : "keyword" + }, + "NewState" : { + "type" : "keyword" + }, + "NewTargetUserName" : { + "type" : "keyword" + }, + "NewTime" : { + "type" : "date" + }, + "NewUacValue" : { + "type" : "keyword" + }, + "NewValue" : { + "type" : "keyword" + }, + "NewValueType" : { + "type" : "keyword" + }, + "NotificationPackageName" : { + "type" : "keyword" + }, + "ObjectClass" : { + "type" : "keyword" + }, + "ObjectCollectionName" : { + "type" : "keyword" + }, + "ObjectDN" : { + "type" : "keyword" + }, + "ObjectGUID" : { + "type" : "keyword" + }, + "ObjectIdentifyingProperties" : { + "type" : "keyword" + }, + "ObjectName" : { + "type" : "keyword" + }, + "ObjectProperties" : { + "type" : "keyword" + }, + "ObjectServer" : { + "type" : "keyword" + }, + "ObjectType" : { + "type" : "keyword" + }, + "ObjectValueName" : { + "type" : "keyword" + }, + "OemInformation" : { + "type" : "keyword" + }, + "OldMaxUsers" : { + "type" : "keyword" + }, + "OldRemark" : { + "type" : "keyword" + }, + "OldSD" : { + "type" : "keyword" + }, + "OldSd" : { + "type" : "keyword" + }, + "OldShareFlags" : { + "type" : "keyword" + }, + "OldTargetUserName" : { + "type" : "keyword" + }, + "OldUacValue" : { + "type" : "keyword" + }, + "OldValue" : { + "type" : "keyword" + }, + "OldValueType" : { + "type" : "keyword" + }, + "OpCorrelationID" : { + "type" : "keyword" + }, + "Operation" : { + "type" : "keyword" + }, + "OperationId" : { + "type" : "keyword" + }, + "OperationMode" : { + "type" : "keyword" + }, + "OperationName" : { + "type" : "keyword" + }, + "OperationType" : { + "type" : "keyword" + }, + "Options" : { + "type" : "keyword" + }, + "OutboundSpi" : { + "type" : "keyword" + }, + "PackageName" : { + "type" : "keyword" + }, + "ParentProcessName" : { + "type" : "keyword" + }, + "PasswordHistoryLength" : { + "type" : "keyword" + }, + "PasswordLastSet" : { + "type" : "keyword" + }, + "PasswordProperties" : { + "type" : "keyword" + }, + "PeerMac" : { + "type" : "keyword" + }, + "PeerPrivateAddress" : { + "type" : "keyword" + }, + "Policy" : { + "type" : "keyword" + }, + "PreAuthType" : { + "type" : "keyword" + }, + "PreviousTime" : { + "type" : "date" + }, + "PrimaryGroupId" : { + "type" : "keyword" + }, + "PrivilegeList" : { + "type" : "keyword" + }, + "ProcessID" : { + "type" : "keyword" + }, + "ProcessId" : { + "type" : "keyword" + }, + "ProcessName" : { + "type" : "keyword" + }, + "ProductName" : { + "type" : "keyword" + }, + "Profile" : { + "type" : "keyword" + }, + "ProfileChanged" : { + "type" : "keyword" + }, + "ProfilePath" : { + "type" : "keyword" + }, + "ProfileUsed" : { + "type" : "keyword" + }, + "Profiles" : { + "type" : "keyword" + }, + "Properties" : { + "type" : "keyword" + }, + "Protocol" : { + "type" : "keyword" + }, + "ProviderContextKey" : { + "type" : "keyword" + }, + "ProviderContextName" : { + "type" : "keyword" + }, + "ProviderContextType" : { + "type" : "keyword" + }, + "ProviderKey" : { + "type" : "keyword" + }, + "ProviderName" : { + "type" : "keyword" + }, + "ProviderType" : { + "type" : "keyword" + }, + "ProxyPolicyName" : { + "type" : "keyword" + }, + "PuaCount" : { + "type" : "keyword" + }, + "PuaPolicyId" : { + "type" : "keyword" + }, + "PublisherGuid" : { + "type" : "keyword" + }, + "PublisherID" : { + "type" : "keyword" + }, + "PublisherName" : { + "type" : "keyword" + }, + "QMFilterID" : { + "type" : "keyword" + }, + "QMLimit" : { + "type" : "keyword" + }, + "QuarantineHelpURL" : { + "type" : "keyword" + }, + "QuarantineSessionID" : { + "type" : "keyword" + }, + "QuarantineSessionIdentifier" : { + "type" : "keyword" + }, + "QuarantineState" : { + "type" : "keyword" + }, + "QuarantineSystemHealthResult" : { + "type" : "keyword" + }, + "QuickModeSaId" : { + "type" : "keyword" + }, + "Reason" : { + "type" : "keyword" + }, + "ReasonCode" : { + "type" : "keyword" + }, + "ReasonForRejection" : { + "type" : "keyword" + }, + "ReasonText" : { + "type" : "keyword" + }, + "RecoveryKeyId" : { + "type" : "keyword" + }, + "RecoveryReason" : { + "type" : "keyword" + }, + "RecoveryServer" : { + "type" : "keyword" + }, + "RelativeTargetName" : { + "type" : "keyword" + }, + "RemoteAddress" : { + "type" : "keyword" + }, + "RemoteAddressMask" : { + "type" : "keyword" + }, + "RemoteAdminEnabled" : { + "type" : "keyword" + }, + "RemoteEventLogging" : { + "type" : "keyword" + }, + "RemoteKeyModPort" : { + "type" : "keyword" + }, + "RemoteMMPrincipalName" : { + "type" : "keyword" + }, + "RemoteMachineID" : { + "type" : "keyword" + }, + "RemotePort" : { + "type" : "keyword" + }, + "RemotePrivateAddress" : { + "type" : "keyword" + }, + "RemoteTunnelEndpoint" : { + "type" : "keyword" + }, + "RemoteUserID" : { + "type" : "keyword" + }, + "RequestId" : { + "type" : "keyword" + }, + "RequestType" : { + "type" : "keyword" + }, + "Requester" : { + "type" : "keyword" + }, + "ResourceAttributes" : { + "type" : "keyword" + }, + "ResourceManager" : { + "type" : "keyword" + }, + "ResponderCookie" : { + "type" : "keyword" + }, + "RestrictedAdminMode" : { + "type" : "keyword" + }, + "RestrictedSidCount" : { + "type" : "keyword" + }, + "ReturnCode" : { + "type" : "keyword" + }, + "Role" : { + "type" : "keyword" + }, + "RuleAttr" : { + "type" : "keyword" + }, + "RuleId" : { + "type" : "keyword" + }, + "RuleName" : { + "type" : "keyword" + }, + "SPI" : { + "type" : "keyword" + }, + "SSID" : { + "type" : "keyword" + }, + "SamAccountName" : { + "type" : "keyword" + }, + "ScopeName" : { + "type" : "keyword" + }, + "ScriptPath" : { + "type" : "keyword" + }, + "SecurityDescriptor" : { + "type" : "keyword" + }, + "SecurityPackageName" : { + "type" : "keyword" + }, + "Service" : { + "type" : "keyword" + }, + "ServiceAccount" : { + "type" : "keyword" + }, + "ServiceFileName" : { + "type" : "keyword" + }, + "ServiceName" : { + "type" : "keyword" + }, + "ServicePrincipalNames" : { + "type" : "keyword" + }, + "ServiceSid" : { + "type" : "keyword" + }, + "ServiceStartType" : { + "type" : "keyword" + }, + "ServiceType" : { + "type" : "keyword" + }, + "SessionID" : { + "type" : "keyword" + }, + "SessionId" : { + "type" : "keyword" + }, + "SessionName" : { + "type" : "keyword" + }, + "SettingType" : { + "type" : "keyword" + }, + "SettingValue" : { + "type" : "keyword" + }, + "ShareLocalPath" : { + "type" : "keyword" + }, + "ShareName" : { + "type" : "keyword" + }, + "SidHistory" : { + "type" : "keyword" + }, + "SourceAddr" : { + "type" : "keyword" + }, + "SourceAddress" : { + "type" : "keyword" + }, + "SourceDRA" : { + "type" : "keyword" + }, + "SourceHandleId" : { + "type" : "keyword" + }, + "SourcePort" : { + "type" : "keyword" + }, + "SourceProcessId" : { + "type" : "keyword" + }, + "StartUSN" : { + "type" : "keyword" + }, + "State" : { + "type" : "keyword" + }, + "Status" : { + "type" : "keyword" + }, + "StatusCode" : { + "type" : "keyword" + }, + "StoreUrl" : { + "type" : "keyword" + }, + "SubLayerKey" : { + "type" : "keyword" + }, + "SubLayerName" : { + "type" : "keyword" + }, + "SubLayerType" : { + "type" : "keyword" + }, + "SubStatus" : { + "type" : "keyword" + }, + "SubcategoryGuid" : { + "type" : "keyword" + }, + "SubcategoryId" : { + "type" : "keyword" + }, + "Subject" : { + "type" : "keyword" + }, + "SubjectDomainName" : { + "type" : "keyword" + }, + "SubjectKeyIdentifier" : { + "type" : "keyword" + }, + "SubjectLogonId" : { + "type" : "keyword" + }, + "SubjectMachineName" : { + "type" : "keyword" + }, + "SubjectMachineSID" : { + "type" : "keyword" + }, + "SubjectUserDomainName" : { + "type" : "keyword" + }, + "SubjectUserName" : { + "type" : "keyword" + }, + "SubjectUserSid" : { + "type" : "keyword" + }, + "TargetDomainName" : { + "type" : "keyword" + }, + "TargetHandleId" : { + "type" : "keyword" + }, + "TargetInfo" : { + "type" : "keyword" + }, + "TargetLinkedLogonId" : { + "type" : "keyword" + }, + "TargetLogonGuid" : { + "type" : "keyword" + }, + "TargetLogonId" : { + "type" : "keyword" + }, + "TargetOutboundDomainName" : { + "type" : "keyword" + }, + "TargetOutboundUserName" : { + "type" : "keyword" + }, + "TargetProcessId" : { + "type" : "keyword" + }, + "TargetServerName" : { + "type" : "keyword" + }, + "TargetSid" : { + "type" : "keyword" + }, + "TargetUserName" : { + "type" : "keyword" + }, + "TargetUserSid" : { + "type" : "keyword" + }, + "TaskContent" : { + "type" : "keyword" + }, + "TaskContentNew" : { + "type" : "keyword" + }, + "TaskName" : { + "type" : "keyword" + }, + "TemplateContent" : { + "type" : "keyword" + }, + "TemplateDSObjectFQDN" : { + "type" : "keyword" + }, + "TemplateInternalName" : { + "type" : "keyword" + }, + "TemplateOID" : { + "type" : "keyword" + }, + "TemplateSchemaVersion" : { + "type" : "keyword" + }, + "TemplateVersion" : { + "type" : "keyword" + }, + "TestSigning" : { + "type" : "keyword" + }, + "TicketEncryptionType" : { + "type" : "keyword" + }, + "TicketOptions" : { + "type" : "keyword" + }, + "TokenElevationType" : { + "type" : "keyword" + }, + "TrafficSelectorId" : { + "type" : "keyword" + }, + "TransactionId" : { + "type" : "keyword" + }, + "TransmittedServices" : { + "type" : "keyword" + }, + "TransportFilterId" : { + "type" : "keyword" + }, + "TreeDelete" : { + "type" : "keyword" + }, + "TunnelId" : { + "type" : "keyword" + }, + "UserAccountControl" : { + "type" : "keyword" + }, + "UserName" : { + "type" : "keyword" + }, + "UserParameters" : { + "type" : "keyword" + }, + "UserPrincipalName" : { + "type" : "keyword" + }, + "UserSid" : { + "type" : "keyword" + }, + "UserWorkstations" : { + "type" : "keyword" + }, + "VirtualAccount" : { + "type" : "keyword" + }, + "VsmLaunchType" : { + "type" : "keyword" + }, + "Weight" : { + "type" : "keyword" + }, + "Workstation" : { + "type" : "keyword" + }, + "WorkstationName" : { + "type" : "keyword" + }, + "param1" : { + "type" : "keyword" + }, + "param2" : { + "type" : "keyword" + }, + "param3" : { + "type" : "keyword" + }, + "param4" : { + "type" : "keyword" + }, + "param5" : { + "type" : "keyword" + }, + "param6" : { + "type" : "keyword" + }, + "param7" : { + "type" : "keyword" + }, + "param8" : { + "type" : "keyword" + }, + "param9" : { + "type" : "keyword" + }, + "xml_name" : { + "type" : "keyword" + } + } + }, + "eventlog" : { + "properties" : { + "computer" : { + "type" : "keyword" + }, + "eventID" : { + "type" : "long" + }, + "event_recordID" : { + "type" : "long" + }, + "keywords" : { + "type" : "keyword" + }, + "log_name" : { + "type" : "keyword" + }, + "processID" : { + "type" : "long" + }, + "source" : { + "type" : "keyword" + }, + "task_category" : { + "type" : "keyword" + }, + "threadID" : { + "type" : "long" + } + } + }, + "hostIP" : { + "type" : "ip" + }, + "hostIP_geo" : { + "dynamic" : "true", + "properties" : { + "location" : { + "type" : "geo_point" + }, + "postal_code" : { + "type" : "keyword" + } + } + }, + "hostname" : { + "type" : "keyword" + }, + "message" : { + "type" : "text", + "norms" : false, + "fields" : { + "raw" : { + "type" : "keyword", + "ignore_above" : 512 + } + } + }, + "message_error" : { + "type" : "text", + "norms" : false, + "fields" : { + "raw" : { + "type" : "keyword", + "ignore_above" : 512 + } + } + }, + "meta" : { + "include_in_all" : false, + "properties" : { + "beat_type" : { + "type" : "keyword" + }, + "beat_version" : { + "type" : "keyword" + }, + "diff_indexer_shipper_ms" : { + "type" : "long" + }, + "diff_shipper_timestamp_ms" : { + "type" : "long" + }, + "kafka_topic" : { + "type" : "keyword" + }, + "processed_at_indexer" : { + "type" : "keyword" + }, + "processed_at_shipper" : { + "type" : "keyword" + }, + "received_at_indexer" : { + "type" : "date" + }, + "received_at_shipper" : { + "type" : "date" + }, + "received_from" : { + "type" : "keyword" + } + } + }, + "net" : { + "properties" : { + "dst_ip" : { + "type" : "ip" + }, + "dst_ip_geo" : { + "properties" : { + "location" : { + "type" : "geo_point" + } + } + }, + "dst_nat_ip" : { + "type" : "ip" + }, + "dst_nat_port" : { + "type" : "long" + }, + "dst_port" : { + "type" : "long" + }, + "src_domain" : { + "type" : "keyword" + }, + "src_ip" : { + "type" : "ip" + }, + "src_ip_geo" : { + "properties" : { + "location" : { + "type" : "geo_point" + } + } + }, + "src_nat_ip" : { + "type" : "ip" + }, + "src_nat_port" : { + "type" : "long" + }, + "src_port" : { + "type" : "long" + }, + "src_user" : { + "type" : "keyword" + }, + "src_user_department" : { + "type" : "keyword" + }, + "src_user_department_number" : { + "type" : "keyword" + }, + "src_user_display_name" : { + "type" : "keyword" + }, + "src_user_title" : { + "type" : "keyword" + } + } + }, + "severity" : { + "type" : "keyword" + }, + "tags" : { + "type" : "text", + "norms" : false, + "fields" : { + "raw" : { + "type" : "keyword", + "ignore_above" : 512 + } + } + }, + "type" : { + "type" : "keyword" + }, + "user" : { + "type" : "object" + } +} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java b/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java deleted file mode 100644 index 968262448c87e..0000000000000 --- a/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos; - -import com.carrotsearch.randomizedtesting.RandomizedTest; -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.apache.http.HttpHost; -import org.elasticsearch.Version; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.junit.Before; - -import java.io.IOException; - -/** - * Tests doc-value-based searches against indices imported from clusters older than N-1. - * We reuse the YAML tests in search/390_doc_values_search.yml but have to do the setup - * manually here as the setup is done on the old cluster for which we have to use the - * low-level REST client instead of the YAML set up that only knows how to talk to - * newer ES versions. - * - * We mimic the setup in search/390_doc_values_search.yml here, but adapt it to work - * against older version clusters. - */ -public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase { - - final Version oldVersion = Version.fromString(System.getProperty("tests.es.version")); - static boolean setupDone; - - public DocValueOnlyFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @ParametersFactory - public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(); - } - - @Override - protected boolean preserveClusterUponCompletion() { - return true; - } - - @Override - protected Settings restClientSettings() { - String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - - @Override - protected boolean skipSetupSections() { - // setup in the YAML file is replaced by the method below - return true; - } - - @Before - public void setupIndex() throws IOException { - final boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); - if (afterRestart) { - return; - } - - // The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only - // initialized later, so we do it when running the first test - if (setupDone) { - return; - } - - setupDone = true; - - String repoLocation = PathUtils.get(System.getProperty("tests.repo.location")) - .resolve(RandomizedTest.getContext().getTargetClass().getName()) - .toString(); - - String indexName = "test"; - String repoName = "doc_values_repo"; - String snapshotName = "snap"; - String[] basicTypes = new String[] { - "byte", - "double", - "float", - "half_float", - "integer", - "long", - "short", - "boolean", - "keyword", - "ip", - "geo_point" }; // date is manually added as it need further configuration - - int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); - try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { - Request createIndex = new Request("PUT", "/" + indexName); - int numberOfShards = randomIntBetween(1, 3); - - boolean multiTypes = oldVersion.before(Version.V_7_0_0); - - XContentBuilder settingsBuilder = XContentFactory.jsonBuilder() - .startObject() - .startObject("settings") - .field("index.number_of_shards", numberOfShards) - .endObject() - .startObject("mappings"); - if (multiTypes) { - settingsBuilder.startObject("doc"); - } - settingsBuilder.field("dynamic", false).startObject("properties"); - for (String type : basicTypes) { - settingsBuilder.startObject(type).field("type", type).endObject(); - } - settingsBuilder.startObject("date").field("type", "date").field("format", "yyyy/MM/dd").endObject(); - if (multiTypes) { - settingsBuilder.endObject(); - } - settingsBuilder.endObject().endObject().endObject(); - - createIndex.setJsonEntity(Strings.toString(settingsBuilder)); - assertOK(oldEs.performRequest(createIndex)); - - Request doc1 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "1"); - doc1.addParameter("refresh", "true"); - XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() - .startObject() - .field("byte", 1) - .field("double", 1.0) - .field("float", 1.0) - .field("half_float", 1.0) - .field("integer", 1) - .field("long", 1) - .field("short", 1) - .field("date", "2017/01/01") - .field("keyword", "key1") - .field("boolean", false) - .field("ip", "192.168.0.1") - .array("geo_point", 13.5, 34.89) - .endObject(); - doc1.setJsonEntity(Strings.toString(bodyDoc1)); - assertOK(oldEs.performRequest(doc1)); - - Request doc2 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "2"); - doc2.addParameter("refresh", "true"); - XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() - .startObject() - .field("byte", 2) - .field("double", 2.0) - .field("float", 2.0) - .field("half_float", 2.0) - .field("integer", 2) - .field("long", 2) - .field("short", 2) - .field("date", "2017/01/02") - .field("keyword", "key2") - .field("boolean", true) - .field("ip", "192.168.0.2") - .array("geo_point", -63.24, 31.0) - .endObject(); - doc2.setJsonEntity(Strings.toString(bodyDoc2)); - assertOK(oldEs.performRequest(doc2)); - - // register repo on old ES and take snapshot - Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest.setJsonEntity(Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(oldEs.performRequest(createRepoRequest)); - - Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); - createSnapshotRequest.addParameter("wait_for_completion", "true"); - createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); - assertOK(oldEs.performRequest(createSnapshotRequest)); - } - - // register repo on new ES and restore snapshot - Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest2.setJsonEntity(Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(client().performRequest(createRepoRequest2)); - - final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); - createRestoreRequest.addParameter("wait_for_completion", "true"); - createRestoreRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); - assertOK(client().performRequest(createRestoreRequest)); - } -} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java deleted file mode 100644 index 67dbdec6b8399..0000000000000 --- a/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java +++ /dev/null @@ -1,361 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos; - -import com.carrotsearch.randomizedtesting.RandomizedTest; - -import org.apache.http.HttpHost; -import org.elasticsearch.Version; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.WarningsHandler; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.hasSize; - -public class OldMappingsIT extends ESRestTestCase { - - static final Version oldVersion = Version.fromString(System.getProperty("tests.es.version")); - - static boolean setupDone; - - @Override - protected boolean preserveClusterUponCompletion() { - return true; - } - - @Override - protected Settings restClientSettings() { - String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - - @Before - public void setupIndex() throws IOException { - final boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); - if (afterRestart) { - return; - } - - // The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only - // initialized later, so we do it when running the first test - if (setupDone) { - return; - } - - setupDone = true; - - String repoLocation = PathUtils.get(System.getProperty("tests.repo.location")) - .resolve(RandomizedTest.getContext().getTargetClass().getName()) - .toString(); - - String repoName = "old_mappings_repo"; - String snapshotName = "snap"; - List indices; - if (oldVersion.before(Version.fromString("6.0.0"))) { - indices = Arrays.asList("filebeat", "winlogbeat", "custom", "nested"); - } else { - indices = Arrays.asList("filebeat", "custom", "nested"); - } - - int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); - try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { - - assertOK(oldEs.performRequest(createIndex("filebeat", "filebeat.json"))); - if (oldVersion.before(Version.fromString("6.0.0"))) { - assertOK(oldEs.performRequest(createIndex("winlogbeat", "winlogbeat.json"))); - } - assertOK(oldEs.performRequest(createIndex("custom", "custom.json"))); - assertOK(oldEs.performRequest(createIndex("nested", "nested.json"))); - - Request doc1 = new Request("PUT", "/" + "custom" + "/" + "doc" + "/" + "1"); - doc1.addParameter("refresh", "true"); - XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() - .startObject() - .startObject("apache2") - .startObject("access") - .field("url", "myurl1") - .field("agent", "agent1") - .endObject() - .endObject() - .endObject(); - doc1.setJsonEntity(Strings.toString(bodyDoc1)); - assertOK(oldEs.performRequest(doc1)); - - Request doc2 = new Request("PUT", "/" + "custom" + "/" + "doc" + "/" + "2"); - doc2.addParameter("refresh", "true"); - XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() - .startObject() - .startObject("apache2") - .startObject("access") - .field("url", "myurl2") - .field("agent", "agent2 agent2") - .endObject() - .endObject() - .field("completion", "some_value") - .endObject(); - doc2.setJsonEntity(Strings.toString(bodyDoc2)); - assertOK(oldEs.performRequest(doc2)); - - Request doc3 = new Request("PUT", "/" + "nested" + "/" + "doc" + "/" + "1"); - doc3.addParameter("refresh", "true"); - XContentBuilder bodyDoc3 = XContentFactory.jsonBuilder() - .startObject() - .field("group", "fans") - .startArray("user") - .startObject() - .field("first", "John") - .field("last", "Smith") - .endObject() - .startObject() - .field("first", "Alice") - .field("last", "White") - .endObject() - .endArray() - .endObject(); - doc3.setJsonEntity(Strings.toString(bodyDoc3)); - assertOK(oldEs.performRequest(doc3)); - - // register repo on old ES and take snapshot - Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest.setJsonEntity(Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(oldEs.performRequest(createRepoRequest)); - - Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); - createSnapshotRequest.addParameter("wait_for_completion", "true"); - createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); - assertOK(oldEs.performRequest(createSnapshotRequest)); - } - - // register repo on new ES and restore snapshot - Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest2.setJsonEntity(Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(client().performRequest(createRepoRequest2)); - - final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); - createRestoreRequest.addParameter("wait_for_completion", "true"); - createRestoreRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); - createRestoreRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); - assertOK(client().performRequest(createRestoreRequest)); - } - - private Request createIndex(String indexName, String file) throws IOException { - Request createIndex = new Request("PUT", "/" + indexName); - int numberOfShards = randomIntBetween(1, 3); - - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("settings") - .field("index.number_of_shards", numberOfShards) - .endObject() - .startObject("mappings"); - builder.rawValue(OldMappingsIT.class.getResourceAsStream(file), XContentType.JSON); - builder.endObject().endObject(); - - createIndex.setJsonEntity(Strings.toString(builder)); - return createIndex; - } - - public void testMappingOk() throws IOException { - Request mappingRequest = new Request("GET", "/" + "filebeat" + "/_mapping"); - Map mapping = entityAsMap(client().performRequest(mappingRequest)); - assertNotNull(XContentMapValues.extractValue(mapping, "filebeat", "mappings", "properties", "apache2")); - - if (oldVersion.before(Version.fromString("6.0.0"))) { - mappingRequest = new Request("GET", "/" + "winlogbeat" + "/_mapping"); - mapping = entityAsMap(client().performRequest(mappingRequest)); - assertNotNull(XContentMapValues.extractValue(mapping, "winlogbeat", "mappings", "properties", "message")); - } - } - - public void testSearchKeyword() throws IOException { - Request search = new Request("POST", "/" + "custom" + "/_search"); - XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("query") - .startObject("match") - .startObject("apache2.access.url") - .field("query", "myurl2") - .endObject() - .endObject() - .endObject() - .endObject(); - search.setJsonEntity(Strings.toString(query)); - Map response = entityAsMap(client().performRequest(search)); - List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); - assertThat(hits, hasSize(1)); - } - - public void testSearchOnPlaceHolderField() throws IOException { - Request search = new Request("POST", "/" + "custom" + "/_search"); - XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("query") - .startObject("match") - .startObject("completion") - .field("query", "some-agent") - .endObject() - .endObject() - .endObject() - .endObject(); - search.setJsonEntity(Strings.toString(query)); - ResponseException re = expectThrows(ResponseException.class, () -> entityAsMap(client().performRequest(search))); - assertThat( - re.getMessage(), - containsString("Field [completion] of type [completion] in legacy index does not support match queries") - ); - } - - public void testAggregationOnPlaceholderField() throws IOException { - Request search = new Request("POST", "/" + "custom" + "/_search"); - XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("aggs") - .startObject("agents") - .startObject("terms") - .field("field", "completion") - .endObject() - .endObject() - .endObject() - .endObject(); - search.setJsonEntity(Strings.toString(query)); - ResponseException re = expectThrows(ResponseException.class, () -> entityAsMap(client().performRequest(search))); - assertThat(re.getMessage(), containsString("can't run aggregation or sorts on field type completion of legacy index")); - } - - public void testConstantScoringOnTextField() throws IOException { - Request search = new Request("POST", "/" + "custom" + "/_search"); - XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("query") - .startObject("match") - .startObject("apache2.access.agent") - .field("query", "agent2") - .endObject() - .endObject() - .endObject() - .endObject(); - search.setJsonEntity(Strings.toString(query)); - Map response = entityAsMap(client().performRequest(search)); - List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); - assertThat(hits, hasSize(1)); - @SuppressWarnings("unchecked") - Map hit = (Map) hits.get(0); - assertThat(hit, hasKey("_score")); - assertEquals(1.0d, (double) hit.get("_score"), 0.01d); - } - - public void testFieldsExistQueryOnTextField() throws IOException { - Request search = new Request("POST", "/" + "custom" + "/_search"); - XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("query") - .startObject("exists") - .field("field", "apache2.access.agent") - .endObject() - .endObject() - .endObject(); - search.setJsonEntity(Strings.toString(query)); - Map response = entityAsMap(client().performRequest(search)); - List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); - assertThat(hits, hasSize(2)); - } - - public void testSearchFieldsOnPlaceholderField() throws IOException { - Request search = new Request("POST", "/" + "custom" + "/_search"); - XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("query") - .startObject("match") - .startObject("apache2.access.url") - .field("query", "myurl2") - .endObject() - .endObject() - .endObject() - .startArray("fields") - .value("completion") - .endArray() - .endObject(); - search.setJsonEntity(Strings.toString(query)); - Map response = entityAsMap(client().performRequest(search)); - List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); - assertThat(hits, hasSize(1)); - logger.info(hits); - Map fields = (Map) (XContentMapValues.extractValue("fields", (Map) hits.get(0))); - assertEquals(List.of("some_value"), fields.get("completion")); - } - - public void testNestedDocuments() throws IOException { - Request search = new Request("POST", "/" + "nested" + "/_search"); - Map response = entityAsMap(client().performRequest(search)); - logger.info(response); - List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); - assertThat(hits, hasSize(1)); - Map source = (Map) (XContentMapValues.extractValue("_source", (Map) hits.get(0))); - assertEquals("fans", source.get("group")); - - search = new Request("POST", "/" + "nested" + "/_search"); - XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("query") - .startObject("nested") - .field("path", "user") - .startObject("query") - .startObject("bool") - .startArray("must") - .startObject() - .startObject("match") - .field("user.first", "Alice") - .endObject() - .endObject() - .startObject() - .startObject("match") - .field("user.last", "White") - .endObject() - .endObject() - .endArray() - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - search.setJsonEntity(Strings.toString(query)); - response = entityAsMap(client().performRequest(search)); - logger.info(response); - hits = (List) (XContentMapValues.extractValue("hits.hits", response)); - assertThat(hits, hasSize(1)); - source = (Map) (XContentMapValues.extractValue("_source", (Map) hits.get(0))); - assertEquals("fans", source.get("group")); - } - -} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java deleted file mode 100644 index 30ec6630b9618..0000000000000 --- a/x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ /dev/null @@ -1,512 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos; - -import org.apache.http.HttpHost; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.cluster.routing.Murmur3HashFunction; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchResponseUtils; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.snapshots.SnapshotState; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.ObjectPath; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.startsWith; - -public class OldRepositoryAccessIT extends ESRestTestCase { - - @Override - protected boolean preserveClusterUponCompletion() { - return true; - } - - @Override - protected Settings restClientSettings() { - String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - - public void testOldRepoAccess() throws IOException { - runTest(false); - } - - public void testOldSourceOnlyRepoAccess() throws IOException { - runTest(true); - } - - public void runTest(boolean sourceOnlyRepository) throws IOException { - boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); - String repoLocation = System.getProperty("tests.repo.location"); - repoLocation = PathUtils.get(repoLocation).resolve("source_only_" + sourceOnlyRepository).toString(); - Version oldVersion = Version.fromString(System.getProperty("tests.es.version")); - assumeTrue( - "source only repositories only supported since ES 6.5.0", - sourceOnlyRepository == false || oldVersion.onOrAfter(Version.fromString("6.5.0")) - ); - - assertThat("Index version should be added to archive tests", oldVersion, lessThan(Version.V_8_10_0)); - IndexVersion indexVersion = IndexVersion.fromId(oldVersion.id); - - int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); - String indexName; - if (sourceOnlyRepository) { - indexName = "source_only_test_index"; - } else { - indexName = "test_index"; - } - int numDocs = 10; - int extraDocs = 1; - final Set expectedIds = new HashSet<>(); - try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { - if (afterRestart == false) { - beforeRestart( - sourceOnlyRepository, - repoLocation, - oldVersion, - indexVersion, - numDocs, - extraDocs, - expectedIds, - oldEs, - indexName - ); - } else { - afterRestart(indexName); - } - } - } - - private void afterRestart(String indexName) throws IOException { - ensureGreen("restored_" + indexName); - ensureGreen("mounted_full_copy_" + indexName); - ensureGreen("mounted_shared_cache_" + indexName); - } - - private void beforeRestart( - boolean sourceOnlyRepository, - String repoLocation, - Version oldVersion, - IndexVersion indexVersion, - int numDocs, - int extraDocs, - Set expectedIds, - RestClient oldEs, - String indexName - ) throws IOException { - String repoName = "repo_" + indexName; - String snapshotName = "snap_" + indexName; - Request createIndex = new Request("PUT", "/" + indexName); - int numberOfShards = randomIntBetween(1, 3); - - XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject().startObject("settings"); - settingsBuilder.field("index.number_of_shards", numberOfShards); - - // 6.5.0 started using soft-deletes, but it was only enabled by default on 7.0 - if (oldVersion.onOrAfter(Version.fromString("6.5.0")) && oldVersion.before(Version.fromString("7.0.0")) && randomBoolean()) { - settingsBuilder.field("index.soft_deletes.enabled", true); - } - - settingsBuilder.endObject().endObject(); - - createIndex.setJsonEntity(Strings.toString(settingsBuilder)); - assertOK(oldEs.performRequest(createIndex)); - - for (int i = 0; i < numDocs + extraDocs; i++) { - String id = "testdoc" + i; - expectedIds.add(id); - // use multiple types for ES versions < 6.0.0 - String type = getType(oldVersion, id); - Request doc = new Request("PUT", "/" + indexName + "/" + type + "/" + id); - doc.addParameter("refresh", "true"); - doc.setJsonEntity(sourceForDoc(i)); - assertOK(oldEs.performRequest(doc)); - } - - for (int i = 0; i < extraDocs; i++) { - String id = randomFrom(expectedIds); - expectedIds.remove(id); - String type = getType(oldVersion, id); - Request doc = new Request("DELETE", "/" + indexName + "/" + type + "/" + id); - doc.addParameter("refresh", "true"); - oldEs.performRequest(doc); - } - - // register repo on old ES and take snapshot - Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest.setJsonEntity(sourceOnlyRepository ? Strings.format(""" - {"type":"source","settings":{"location":"%s","delegate_type":"fs"}} - """, repoLocation) : Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(oldEs.performRequest(createRepoRequest)); - - Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); - createSnapshotRequest.addParameter("wait_for_completion", "true"); - createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); - assertOK(oldEs.performRequest(createSnapshotRequest)); - - // register repo on new ES - Settings.Builder repoSettingsBuilder = Settings.builder().put("location", repoLocation); - if (sourceOnlyRepository) { - repoSettingsBuilder.put("delegate_type", "fs"); - } - Request createRepo = new Request("PUT", "/_snapshot/" + repoName); - createRepo.setJsonEntity( - Strings.toString( - new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).type(sourceOnlyRepository ? "source" : "fs") - .settings(repoSettingsBuilder.build()) - ) - ); - assertAcknowledged(client().performRequest(createRepo)); - - // list snapshots on new ES - Request getSnaps = new Request("GET", "/_snapshot/" + repoName + "/_all"); - Response getResponse = client().performRequest(getSnaps); - ObjectPath getResp = ObjectPath.createFromResponse(getResponse); - assertThat(getResp.evaluate("total"), equalTo(1)); - assertThat(getResp.evaluate("snapshots.0.snapshot"), equalTo(snapshotName)); - assertThat(getResp.evaluate("snapshots.0.repository"), equalTo(repoName)); - assertThat(getResp.evaluate("snapshots.0.indices"), contains(indexName)); - assertThat(getResp.evaluate("snapshots.0.state"), equalTo(SnapshotState.SUCCESS.toString())); - assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.successful")); - assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.total")); - assertEquals(0, (int) getResp.evaluate("snapshots.0.shards.failed")); - assertEquals(indexVersion.toReleaseVersion(), getResp.evaluate("snapshots.0.version")); - - // list specific snapshot on new ES - getSnaps = new Request("GET", "/_snapshot/" + repoName + "/" + snapshotName); - getResponse = client().performRequest(getSnaps); - getResp = ObjectPath.createFromResponse(getResponse); - assertThat(getResp.evaluate("total"), equalTo(1)); - assertThat(getResp.evaluate("snapshots.0.snapshot"), equalTo(snapshotName)); - assertThat(getResp.evaluate("snapshots.0.repository"), equalTo(repoName)); - assertThat(getResp.evaluate("snapshots.0.indices"), contains(indexName)); - assertThat(getResp.evaluate("snapshots.0.state"), equalTo(SnapshotState.SUCCESS.toString())); - assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.successful")); - assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.total")); - assertEquals(0, (int) getResp.evaluate("snapshots.0.shards.failed")); - assertEquals(indexVersion.toReleaseVersion(), getResp.evaluate("snapshots.0.version")); - - // list advanced snapshot info on new ES - getSnaps = new Request("GET", "/_snapshot/" + repoName + "/" + snapshotName + "/_status"); - getResponse = client().performRequest(getSnaps); - getResp = ObjectPath.createFromResponse(getResponse); - assertThat(((List) getResp.evaluate("snapshots")).size(), equalTo(1)); - assertThat(getResp.evaluate("snapshots.0.snapshot"), equalTo(snapshotName)); - assertThat(getResp.evaluate("snapshots.0.repository"), equalTo(repoName)); - assertThat(((Map) getResp.evaluate("snapshots.0.indices")).keySet(), contains(indexName)); - assertThat(getResp.evaluate("snapshots.0.state"), equalTo(SnapshotState.SUCCESS.toString())); - assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards_stats.done")); - assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards_stats.total")); - assertEquals(0, (int) getResp.evaluate("snapshots.0.shards_stats.failed")); - assertThat(getResp.evaluate("snapshots.0.stats.total.size_in_bytes"), greaterThan(0)); - assertThat(getResp.evaluate("snapshots.0.stats.total.file_count"), greaterThan(0)); - - // restore / mount and check whether searches work - restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); - - // close indices - closeIndex(client(), "restored_" + indexName); - closeIndex(client(), "mounted_full_copy_" + indexName); - closeIndex(client(), "mounted_shared_cache_" + indexName); - - // restore / mount again - restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); - } - - private String getType(Version oldVersion, String id) { - return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Math.abs(Murmur3HashFunction.hash(id) % 2) : 0); - } - - private static String sourceForDoc(int i) { - return "{\"test\":\"test" + i + "\",\"val\":" + i + ",\"create_date\":\"2020-01-" + Strings.format("%02d", i + 1) + "\"}"; - } - - private void restoreMountAndVerify( - int numDocs, - Set expectedIds, - int numberOfShards, - boolean sourceOnlyRepository, - Version oldVersion, - String indexName, - String repoName, - String snapshotName - ) throws IOException { - // restore index - Request restoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); - restoreRequest.setJsonEntity( - Strings.toString( - new RestoreSnapshotRequest(TEST_REQUEST_TIMEOUT).indices(indexName).renamePattern("(.+)").renameReplacement("restored_$1") - ) - ); - restoreRequest.addParameter("wait_for_completion", "true"); - Response restoreResponse = client().performRequest(restoreRequest); - ObjectPath restore = ObjectPath.createFromResponse(restoreResponse); - assertEquals(numberOfShards, (int) restore.evaluate("snapshot.shards.total")); - assertEquals(numberOfShards, (int) restore.evaluate("snapshot.shards.successful")); - - ensureGreen("restored_" + indexName); - - String restoredIndex = "restored_" + indexName; - var response = responseAsMap(client().performRequest(new Request("GET", "/" + restoredIndex + "/_mapping"))); - Map mapping = ObjectPath.evaluate(response, restoredIndex + ".mappings"); - logger.info("mapping for {}: {}", restoredIndex, mapping); - assertThat(mapping, hasKey("_meta")); - assertThat(mapping.get("_meta"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map meta = (Map) mapping.get("_meta"); - assertThat(meta, hasKey("legacy_mappings")); - assertThat(meta.get("legacy_mappings"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map legacyMappings = (Map) meta.get("legacy_mappings"); - assertThat(legacyMappings.keySet(), not(empty())); - for (Map.Entry entry : legacyMappings.entrySet()) { - String type = entry.getKey(); - assertThat(type, startsWith("doc")); - assertThat(entry.getValue(), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map legacyMapping = (Map) entry.getValue(); - assertThat(legacyMapping, hasKey("properties")); - assertThat(legacyMapping.get("properties"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map propertiesMapping = (Map) legacyMapping.get("properties"); - assertThat(propertiesMapping, hasKey("val")); - assertThat(propertiesMapping.get("val"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map valMapping = (Map) propertiesMapping.get("val"); - assertThat(valMapping, hasKey("type")); - assertEquals("long", valMapping.get("type")); - } - - // run a search against the index - assertDocs("restored_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); - - // mount as full copy searchable snapshot - Request mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); - mountRequest.setJsonEntity( - "{\"index\": \"" - + indexName - + "\",\"renamed_index\": \"mounted_full_copy_" - + indexName - + "\",\"index_settings\": {\"index.number_of_replicas\": 1}}" - ); - mountRequest.addParameter("wait_for_completion", "true"); - ObjectPath mountResponse = ObjectPath.createFromResponse(client().performRequest(mountRequest)); - assertNotNull(mountResponse.evaluate("snapshot")); - assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.total")); - assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.successful")); - - ensureGreen("mounted_full_copy_" + indexName); - - // run a search against the index - assertDocs("mounted_full_copy_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); - - // mount as shared cache searchable snapshot - mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); - mountRequest.setJsonEntity("{\"index\": \"" + indexName + "\",\"renamed_index\": \"mounted_shared_cache_" + indexName + "\"}"); - mountRequest.addParameter("wait_for_completion", "true"); - mountRequest.addParameter("storage", "shared_cache"); - mountResponse = ObjectPath.createFromResponse(client().performRequest(mountRequest)); - assertNotNull(mountResponse.evaluate("snapshot")); - assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.total")); - assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.successful")); - - // run a search against the index - assertDocs("mounted_shared_cache_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); - } - - private void assertDocs( - String index, - int numDocs, - Set expectedIds, - boolean sourceOnlyRepository, - Version oldVersion, - int numberOfShards - ) throws IOException { - RequestOptions requestOptions = RequestOptions.DEFAULT; - - // run a search against the index - SearchResponse searchResponse = search(index, null, requestOptions); - try { - logger.info(searchResponse); - // check hit count - assertEquals(numDocs, searchResponse.getHits().getTotalHits().value()); - // check that _index is properly set - assertTrue(Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getIndex).allMatch(index::equals)); - // check that all _ids are there - assertEquals(expectedIds, Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toSet())); - // check that _source is present - assertTrue(Arrays.stream(searchResponse.getHits().getHits()).allMatch(SearchHit::hasSource)); - // check that correct _source present for each document - for (SearchHit h : searchResponse.getHits().getHits()) { - assertEquals(sourceForDoc(getIdAsNumeric(h.getId())), h.getSourceAsString()); - } - } finally { - searchResponse.decRef(); - } - - String id = randomFrom(expectedIds); - int num = getIdAsNumeric(id); - // run a search using runtime fields against the index - searchResponse = search( - index, - SearchSourceBuilder.searchSource() - .query(QueryBuilders.matchQuery("val", num)) - .runtimeMappings(Map.of("val", Map.of("type", "long"))), - requestOptions - ); - try { - logger.info(searchResponse); - assertEquals(1, searchResponse.getHits().getTotalHits().value()); - assertEquals(id, searchResponse.getHits().getHits()[0].getId()); - assertEquals(sourceForDoc(num), searchResponse.getHits().getHits()[0].getSourceAsString()); - } finally { - searchResponse.decRef(); - } - - if (sourceOnlyRepository == false) { - // search using reverse sort on val - searchResponse = search( - index, - SearchSourceBuilder.searchSource() - .query(QueryBuilders.matchAllQuery()) - .sort(SortBuilders.fieldSort("val").order(SortOrder.DESC)), - requestOptions - ); - try { - logger.info(searchResponse); - // check sort order - assertEquals( - expectedIds.stream().sorted(Comparator.comparingInt(this::getIdAsNumeric).reversed()).toList(), - Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).toList() - ); - } finally { - searchResponse.decRef(); - } - - // look up postings - searchResponse = search( - index, - SearchSourceBuilder.searchSource().query(QueryBuilders.matchQuery("test", "test" + num)), - requestOptions - ); - try { - logger.info(searchResponse); - // check match - ElasticsearchAssertions.assertSearchHits(searchResponse, id); - } finally { - searchResponse.decRef(); - } - - if (oldVersion.before(Version.fromString("6.0.0"))) { - // search on _type and check that results contain _type information - String randomType = getType(oldVersion, randomFrom(expectedIds)); - long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count(); - searchResponse = search( - index, - SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType)), - requestOptions - ); - try { - logger.info(searchResponse); - assertEquals(typeCount, searchResponse.getHits().getTotalHits().value()); - for (SearchHit hit : searchResponse.getHits().getHits()) { - DocumentField typeField = hit.field("_type"); - assertNotNull(typeField); - assertThat(typeField.getValue(), instanceOf(String.class)); - assertEquals(randomType, typeField.getValue()); - } - } finally { - searchResponse.decRef(); - } - } - - assertThat( - expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/" + index + "/_doc/" + id))) - .getMessage(), - containsString("get operations not allowed on a legacy index") - ); - - // check that shards are skipped based on non-matching date - searchResponse = search( - index, - SearchSourceBuilder.searchSource().query(QueryBuilders.rangeQuery("create_date").from("2020-02-01")), - requestOptions - ); - try { - logger.info(searchResponse); - assertEquals(0, searchResponse.getHits().getTotalHits().value()); - assertEquals(numberOfShards, searchResponse.getSuccessfulShards()); - assertEquals(numberOfShards, searchResponse.getSkippedShards()); - } finally { - searchResponse.decRef(); - } - } - } - - private static SearchResponse search(String index, @Nullable SearchSourceBuilder builder, RequestOptions options) throws IOException { - Request request = new Request("POST", "/" + index + "/_search"); - if (builder != null) { - request.setJsonEntity(builder.toString()); - } - request.setOptions(options); - return SearchResponseUtils.parseSearchResponse(responseAsParser(client().performRequest(request))); - } - - private int getIdAsNumeric(String id) { - return Integer.parseInt(id.substring("testdoc".length())); - } - - private static void closeIndex(RestClient client, String index) throws IOException { - Request request = new Request("POST", "/" + index + "/_close"); - ObjectPath doc = ObjectPath.createFromResponse(client.performRequest(request)); - assertTrue(doc.evaluate("shards_acknowledged")); - } -} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/custom.json b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/custom.json deleted file mode 100644 index ae52ccbcce330..0000000000000 --- a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/custom.json +++ /dev/null @@ -1,22 +0,0 @@ -"_default_": { - "properties": { - "apache2": { - "properties": { - "access": { - "properties": { - "agent": { - "type": "text" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "completion": { - "type": "completion" - } - } -} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/filebeat.json b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/filebeat.json deleted file mode 100644 index 6fa22f1c36ef9..0000000000000 --- a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/filebeat.json +++ /dev/null @@ -1,681 +0,0 @@ -"_default_": { - "_meta": { - "version": "5.6.17" - }, - "date_detection": false, - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "@timestamp": { - "type": "date" - }, - "apache2": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "response_code": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "client": { - "ignore_above": 1024, - "type": "keyword" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - }, - "tid": { - "type": "long" - } - } - } - } - }, - "auditd": { - "properties": { - "log": { - "properties": { - "a0": { - "ignore_above": 1024, - "type": "keyword" - }, - "acct": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "item": { - "ignore_above": 1024, - "type": "keyword" - }, - "items": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_auid": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_ses": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_auid": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_ses": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ppid": { - "ignore_above": 1024, - "type": "keyword" - }, - "record_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "res": { - "ignore_above": 1024, - "type": "keyword" - }, - "sequence": { - "type": "long" - } - } - } - } - }, - "beat": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "ignore_above": 1024, - "type": "keyword" - }, - "fileset": { - "properties": { - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "input_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "meta": { - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "instance_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "project_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "mysql": { - "properties": { - "error": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "thread_id": { - "type": "long" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "slowlog": { - "properties": { - "host": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "type": "long" - }, - "ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "lock_time": { - "properties": { - "sec": { - "type": "float" - } - } - }, - "query": { - "ignore_above": 1024, - "type": "keyword" - }, - "query_time": { - "properties": { - "sec": { - "type": "float" - } - } - }, - "rows_examined": { - "type": "long" - }, - "rows_sent": { - "type": "long" - }, - "timestamp": { - "type": "long" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "nginx": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "response_code": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "connection_id": { - "type": "long" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "pid": { - "type": "long" - }, - "tid": { - "type": "long" - } - } - } - } - }, - "offset": { - "type": "long" - }, - "read_timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "source": { - "ignore_above": 1024, - "type": "keyword" - }, - "system": { - "properties": { - "auth": { - "properties": { - "groupadd": { - "properties": { - "gid": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - }, - "program": { - "ignore_above": 1024, - "type": "keyword" - }, - "ssh": { - "properties": { - "dropped_ip": { - "type": "ip" - }, - "event": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "port": { - "type": "long" - }, - "signature": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "sudo": { - "properties": { - "command": { - "ignore_above": 1024, - "type": "keyword" - }, - "error": { - "ignore_above": 1024, - "type": "keyword" - }, - "pwd": { - "ignore_above": 1024, - "type": "keyword" - }, - "tty": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "useradd": { - "properties": { - "gid": { - "type": "long" - }, - "home": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "shell": { - "ignore_above": 1024, - "type": "keyword" - }, - "uid": { - "type": "long" - } - } - } - } - }, - "syslog": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "ignore_above": 1024, - "type": "keyword" - }, - "program": { - "ignore_above": 1024, - "type": "keyword" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } -} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/nested.json b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/nested.json deleted file mode 100644 index 46dfb5243f06a..0000000000000 --- a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/nested.json +++ /dev/null @@ -1,7 +0,0 @@ -"_default_": { - "properties": { - "user": { - "type": "nested" - } - } -} diff --git a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/winlogbeat.json b/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/winlogbeat.json deleted file mode 100644 index ee4500e10ccfc..0000000000000 --- a/x-pack/qa/repository-old-versions-7x/src/test/resources/org/elasticsearch/oldrepos/winlogbeat.json +++ /dev/null @@ -1,1612 +0,0 @@ -"_default_" : { - "_all" : { - "enabled" : true, - "norms" : false - }, - "dynamic_templates" : [ - { - "message_field" : { - "path_match" : "*message*", - "match_mapping_type" : "string", - "mapping" : { - "fields" : { - "raw" : { - "ignore_above" : 512, - "type" : "keyword" - } - }, - "norms" : false, - "type" : "text" - } - } - }, - { - "tags_field" : { - "path_match" : "*tags*", - "match_mapping_type" : "string", - "mapping" : { - "fields" : { - "raw" : { - "ignore_above" : 512, - "type" : "keyword" - } - }, - "norms" : false, - "type" : "text" - } - } - }, - { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "norms" : false, - "type" : "keyword" - } - } - } - ], - "properties" : { - "@timestamp" : { - "type" : "date", - "include_in_all" : false - }, - "@version" : { - "type" : "keyword", - "include_in_all" : false - }, - "hostIP" : { - "type" : "ip" - }, - "hostIP_geo" : { - "dynamic" : "true", - "properties" : { - "location" : { - "type" : "geo_point" - }, - "postal_code" : { - "type" : "keyword" - } - } - }, - "hostname" : { - "type" : "keyword" - }, - "meta" : { - "include_in_all" : false, - "properties" : { - "processed_at_indexer" : { - "type" : "keyword" - }, - "processed_at_shipper" : { - "type" : "keyword" - }, - "received_from" : { - "type" : "keyword" - } - } - }, - "net" : { - "properties" : { - "dst_ip" : { - "type" : "ip" - }, - "dst_ip_geo" : { - "properties" : { - "location" : { - "type" : "geo_point" - } - } - }, - "dst_nat_ip" : { - "type" : "ip" - }, - "dst_nat_port" : { - "type" : "long" - }, - "dst_port" : { - "type" : "long" - }, - "src_ip" : { - "type" : "ip" - }, - "src_ip_geo" : { - "properties" : { - "location" : { - "type" : "geo_point" - } - } - }, - "src_nat_ip" : { - "type" : "ip" - }, - "src_nat_port" : { - "type" : "long" - }, - "src_port" : { - "type" : "long" - } - } - }, - "type" : { - "type" : "keyword" - } - } -}, -"wineventlog" : { - "_all" : { - "enabled" : true, - "norms" : false - }, - "dynamic_templates" : [ - { - "message_field" : { - "path_match" : "*message*", - "match_mapping_type" : "string", - "mapping" : { - "fields" : { - "raw" : { - "ignore_above" : 512, - "type" : "keyword" - } - }, - "norms" : false, - "type" : "text" - } - } - }, - { - "tags_field" : { - "path_match" : "*tags*", - "match_mapping_type" : "string", - "mapping" : { - "fields" : { - "raw" : { - "ignore_above" : 512, - "type" : "keyword" - } - }, - "norms" : false, - "type" : "text" - } - } - }, - { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "norms" : false, - "type" : "keyword" - } - } - } - ], - "properties" : { - "@timestamp" : { - "type" : "date", - "include_in_all" : false - }, - "@version" : { - "type" : "keyword", - "include_in_all" : false - }, - "beat" : { - "type" : "object" - }, - "eventdata" : { - "properties" : { - "AccessGranted" : { - "type" : "keyword" - }, - "AccessList" : { - "type" : "keyword" - }, - "AccessMask" : { - "type" : "keyword" - }, - "AccessReason" : { - "type" : "keyword" - }, - "AccessRemoved" : { - "type" : "keyword" - }, - "AccountDomain" : { - "type" : "keyword" - }, - "AccountExpires" : { - "type" : "keyword" - }, - "AccountName" : { - "type" : "keyword" - }, - "AccountSessionIdentifier" : { - "type" : "keyword" - }, - "Action" : { - "type" : "keyword" - }, - "ActiveProfile" : { - "type" : "keyword" - }, - "AdditionalInfo" : { - "type" : "keyword" - }, - "AdditionalInfo2" : { - "type" : "keyword" - }, - "AdvancedOptions" : { - "type" : "keyword" - }, - "AhAuthType" : { - "type" : "keyword" - }, - "AlgorithmName" : { - "type" : "keyword" - }, - "AllowedToDelegateTo" : { - "type" : "keyword" - }, - "AppCorrelationID" : { - "type" : "keyword" - }, - "AppInstance" : { - "type" : "keyword" - }, - "AppName" : { - "type" : "keyword" - }, - "Application" : { - "type" : "keyword" - }, - "AttributeLDAPDisplayName" : { - "type" : "keyword" - }, - "AttributeSyntaxOID" : { - "type" : "keyword" - }, - "AttributeValue" : { - "type" : "keyword" - }, - "Attributes" : { - "type" : "keyword" - }, - "AuditPolicyChanges" : { - "type" : "keyword" - }, - "AuditSourceName" : { - "type" : "keyword" - }, - "AuditsDiscarded" : { - "type" : "keyword" - }, - "AuthenticationPackage" : { - "type" : "keyword" - }, - "AuthenticationPackageName" : { - "type" : "keyword" - }, - "AuthenticationProvider" : { - "type" : "keyword" - }, - "AuthenticationServer" : { - "type" : "keyword" - }, - "AuthenticationType" : { - "type" : "keyword" - }, - "BackupPath" : { - "type" : "keyword" - }, - "BackupType" : { - "type" : "keyword" - }, - "CalledStationID" : { - "type" : "keyword" - }, - "CallerProcessId" : { - "type" : "keyword" - }, - "CallerProcessName" : { - "type" : "keyword" - }, - "CallingStationID" : { - "type" : "keyword" - }, - "CalloutId" : { - "type" : "keyword" - }, - "CalloutKey" : { - "type" : "keyword" - }, - "CalloutName" : { - "type" : "keyword" - }, - "CalloutType" : { - "type" : "keyword" - }, - "Categories" : { - "type" : "keyword" - }, - "CategoryId" : { - "type" : "keyword" - }, - "ChangeType" : { - "type" : "keyword" - }, - "Channel" : { - "type" : "keyword" - }, - "CipherType" : { - "type" : "keyword" - }, - "ClientAddress" : { - "type" : "keyword" - }, - "ClientDomain" : { - "type" : "keyword" - }, - "ClientIPAddress" : { - "type" : "keyword" - }, - "ClientLogonId" : { - "type" : "keyword" - }, - "ClientName" : { - "type" : "keyword" - }, - "ClientUserName" : { - "type" : "keyword" - }, - "ComputerAccountChange" : { - "type" : "keyword" - }, - "Conditions" : { - "type" : "keyword" - }, - "ConfigAccessPolicy" : { - "type" : "keyword" - }, - "DCDNSName" : { - "type" : "keyword" - }, - "DHGroup" : { - "type" : "keyword" - }, - "DSName" : { - "type" : "keyword" - }, - "DSType" : { - "type" : "keyword" - }, - "DestAddress" : { - "type" : "keyword" - }, - "DestPort" : { - "type" : "keyword" - }, - "DestinationDRA" : { - "type" : "keyword" - }, - "Direction" : { - "type" : "keyword" - }, - "DisableIntegrityChecks" : { - "type" : "keyword" - }, - "DisabledPrivilegeList" : { - "type" : "keyword" - }, - "DisplayName" : { - "type" : "keyword" - }, - "Disposition" : { - "type" : "keyword" - }, - "DnsHostName" : { - "type" : "keyword" - }, - "DomainBehaviorVersion" : { - "type" : "keyword" - }, - "DomainName" : { - "type" : "keyword" - }, - "DomainPolicyChanged" : { - "type" : "keyword" - }, - "DomainSid" : { - "type" : "keyword" - }, - "Dummy" : { - "type" : "keyword" - }, - "EAPErrorCode" : { - "type" : "keyword" - }, - "EAPReasonCode" : { - "type" : "keyword" - }, - "EAPType" : { - "type" : "keyword" - }, - "ElevatedToken" : { - "type" : "keyword" - }, - "EnabledPrivilegeList" : { - "type" : "keyword" - }, - "EndUSN" : { - "type" : "keyword" - }, - "ErrorCode" : { - "type" : "keyword" - }, - "EspAuthType" : { - "type" : "keyword" - }, - "EventCountTotal" : { - "type" : "keyword" - }, - "EventID" : { - "type" : "keyword" - }, - "EventIdx" : { - "type" : "keyword" - }, - "EventSourceId" : { - "type" : "keyword" - }, - "ExtendedQuarantineState" : { - "type" : "keyword" - }, - "FailureDescription" : { - "type" : "keyword" - }, - "FailureId" : { - "type" : "keyword" - }, - "FailurePoint" : { - "type" : "keyword" - }, - "FailureReason" : { - "type" : "keyword" - }, - "FileName" : { - "type" : "keyword" - }, - "FilterId" : { - "type" : "keyword" - }, - "FilterKey" : { - "type" : "keyword" - }, - "FilterName" : { - "type" : "keyword" - }, - "FilterRTID" : { - "type" : "keyword" - }, - "FilterType" : { - "type" : "keyword" - }, - "FlightSigning" : { - "type" : "keyword" - }, - "ForceLogoff" : { - "type" : "keyword" - }, - "FullyQualifiedSubjectMachineName" : { - "type" : "keyword" - }, - "FullyQualifiedSubjectUserName" : { - "type" : "keyword" - }, - "GPOList" : { - "type" : "keyword" - }, - "Group" : { - "type" : "keyword" - }, - "GroupMembership" : { - "type" : "keyword" - }, - "GroupPolicyApplied" : { - "type" : "keyword" - }, - "HandleId" : { - "type" : "keyword" - }, - "HomeDirectory" : { - "type" : "keyword" - }, - "HomePath" : { - "type" : "keyword" - }, - "HypervisorDebug" : { - "type" : "keyword" - }, - "HypervisorLaunchType" : { - "type" : "keyword" - }, - "HypervisorLoadOptions" : { - "type" : "keyword" - }, - "Identity" : { - "type" : "keyword" - }, - "ImpersonationLevel" : { - "type" : "keyword" - }, - "InboundSpi" : { - "type" : "keyword" - }, - "InitiatorCookie" : { - "type" : "keyword" - }, - "InterfaceName" : { - "type" : "keyword" - }, - "IntfGuid" : { - "type" : "keyword" - }, - "IpAddress" : { - "type" : "keyword" - }, - "IpPort" : { - "type" : "keyword" - }, - "IpProtocol" : { - "type" : "keyword" - }, - "KernelDebug" : { - "type" : "keyword" - }, - "KeyFilePath" : { - "type" : "keyword" - }, - "KeyLength" : { - "type" : "keyword" - }, - "KeyModName" : { - "type" : "keyword" - }, - "KeyName" : { - "type" : "keyword" - }, - "KeyType" : { - "type" : "keyword" - }, - "KeyingModuleName" : { - "type" : "keyword" - }, - "LayerId" : { - "type" : "keyword" - }, - "LayerKey" : { - "type" : "keyword" - }, - "LayerName" : { - "type" : "keyword" - }, - "LayerRTID" : { - "type" : "keyword" - }, - "LifetimeKilobytes" : { - "type" : "keyword" - }, - "LifetimePackets" : { - "type" : "keyword" - }, - "LifetimeSeconds" : { - "type" : "keyword" - }, - "LinkName" : { - "type" : "keyword" - }, - "LmPackageName" : { - "type" : "keyword" - }, - "LoadOptions" : { - "type" : "keyword" - }, - "LocalAddress" : { - "type" : "keyword" - }, - "LocalAddressMask" : { - "type" : "keyword" - }, - "LocalKeyModPort" : { - "type" : "keyword" - }, - "LocalMMPrincipalName" : { - "type" : "keyword" - }, - "LocalMac" : { - "type" : "keyword" - }, - "LocalPort" : { - "type" : "keyword" - }, - "LocalTunnelEndpoint" : { - "type" : "keyword" - }, - "LockoutDuration" : { - "type" : "keyword" - }, - "LockoutObservationWindow" : { - "type" : "keyword" - }, - "LockoutThreshold" : { - "type" : "keyword" - }, - "LogDroppedPacketsEnabled" : { - "type" : "keyword" - }, - "LogSuccessfulConnectionsEnabled" : { - "type" : "keyword" - }, - "LoggingResult" : { - "type" : "keyword" - }, - "LogonGuid" : { - "type" : "keyword" - }, - "LogonHours" : { - "type" : "keyword" - }, - "LogonID" : { - "type" : "keyword" - }, - "LogonProcessName" : { - "type" : "keyword" - }, - "LogonType" : { - "type" : "keyword" - }, - "LogonType_Description" : { - "type" : "keyword" - }, - "MMAuthMethod" : { - "type" : "keyword" - }, - "MMCipherAlg" : { - "type" : "keyword" - }, - "MMFilterID" : { - "type" : "keyword" - }, - "MMImpersonationState" : { - "type" : "keyword" - }, - "MMIntegrityAlg" : { - "type" : "keyword" - }, - "MMLifetime" : { - "type" : "keyword" - }, - "MMSAID" : { - "type" : "keyword" - }, - "MachineAccountQuota" : { - "type" : "keyword" - }, - "MachineInventory" : { - "type" : "keyword" - }, - "MainModeSaId" : { - "type" : "keyword" - }, - "MandatoryLabel" : { - "type" : "keyword" - }, - "MappedName" : { - "type" : "keyword" - }, - "MappingBy" : { - "type" : "keyword" - }, - "MasterKeyId" : { - "type" : "keyword" - }, - "MaxPasswordAge" : { - "type" : "keyword" - }, - "MemberName" : { - "type" : "keyword" - }, - "MemberSid" : { - "type" : "keyword" - }, - "Message" : { - "type" : "keyword" - }, - "MessageID" : { - "type" : "keyword" - }, - "MinPasswordAge" : { - "type" : "keyword" - }, - "MinPasswordLength" : { - "type" : "keyword" - }, - "MixedDomainMode" : { - "type" : "keyword" - }, - "Mode" : { - "type" : "keyword" - }, - "ModifiedObjectProperties" : { - "type" : "keyword" - }, - "Module" : { - "type" : "keyword" - }, - "MulticastFlowsEnabled" : { - "type" : "keyword" - }, - "NASIPv4Address" : { - "type" : "keyword" - }, - "NASIPv6Address" : { - "type" : "keyword" - }, - "NASIdentifier" : { - "type" : "keyword" - }, - "NASPort" : { - "type" : "keyword" - }, - "NASPortType" : { - "type" : "keyword" - }, - "NamingContext" : { - "type" : "keyword" - }, - "NetworkPolicyName" : { - "type" : "keyword" - }, - "NewMaxUsers" : { - "type" : "keyword" - }, - "NewProcessId" : { - "type" : "keyword" - }, - "NewProcessName" : { - "type" : "keyword" - }, - "NewRemark" : { - "type" : "keyword" - }, - "NewSD" : { - "type" : "keyword" - }, - "NewSd" : { - "type" : "keyword" - }, - "NewShareFlags" : { - "type" : "keyword" - }, - "NewState" : { - "type" : "keyword" - }, - "NewTargetUserName" : { - "type" : "keyword" - }, - "NewTime" : { - "type" : "date" - }, - "NewUacValue" : { - "type" : "keyword" - }, - "NewValue" : { - "type" : "keyword" - }, - "NewValueType" : { - "type" : "keyword" - }, - "NotificationPackageName" : { - "type" : "keyword" - }, - "ObjectClass" : { - "type" : "keyword" - }, - "ObjectCollectionName" : { - "type" : "keyword" - }, - "ObjectDN" : { - "type" : "keyword" - }, - "ObjectGUID" : { - "type" : "keyword" - }, - "ObjectIdentifyingProperties" : { - "type" : "keyword" - }, - "ObjectName" : { - "type" : "keyword" - }, - "ObjectProperties" : { - "type" : "keyword" - }, - "ObjectServer" : { - "type" : "keyword" - }, - "ObjectType" : { - "type" : "keyword" - }, - "ObjectValueName" : { - "type" : "keyword" - }, - "OemInformation" : { - "type" : "keyword" - }, - "OldMaxUsers" : { - "type" : "keyword" - }, - "OldRemark" : { - "type" : "keyword" - }, - "OldSD" : { - "type" : "keyword" - }, - "OldSd" : { - "type" : "keyword" - }, - "OldShareFlags" : { - "type" : "keyword" - }, - "OldTargetUserName" : { - "type" : "keyword" - }, - "OldUacValue" : { - "type" : "keyword" - }, - "OldValue" : { - "type" : "keyword" - }, - "OldValueType" : { - "type" : "keyword" - }, - "OpCorrelationID" : { - "type" : "keyword" - }, - "Operation" : { - "type" : "keyword" - }, - "OperationId" : { - "type" : "keyword" - }, - "OperationMode" : { - "type" : "keyword" - }, - "OperationName" : { - "type" : "keyword" - }, - "OperationType" : { - "type" : "keyword" - }, - "Options" : { - "type" : "keyword" - }, - "OutboundSpi" : { - "type" : "keyword" - }, - "PackageName" : { - "type" : "keyword" - }, - "ParentProcessName" : { - "type" : "keyword" - }, - "PasswordHistoryLength" : { - "type" : "keyword" - }, - "PasswordLastSet" : { - "type" : "keyword" - }, - "PasswordProperties" : { - "type" : "keyword" - }, - "PeerMac" : { - "type" : "keyword" - }, - "PeerPrivateAddress" : { - "type" : "keyword" - }, - "Policy" : { - "type" : "keyword" - }, - "PreAuthType" : { - "type" : "keyword" - }, - "PreviousTime" : { - "type" : "date" - }, - "PrimaryGroupId" : { - "type" : "keyword" - }, - "PrivilegeList" : { - "type" : "keyword" - }, - "ProcessID" : { - "type" : "keyword" - }, - "ProcessId" : { - "type" : "keyword" - }, - "ProcessName" : { - "type" : "keyword" - }, - "ProductName" : { - "type" : "keyword" - }, - "Profile" : { - "type" : "keyword" - }, - "ProfileChanged" : { - "type" : "keyword" - }, - "ProfilePath" : { - "type" : "keyword" - }, - "ProfileUsed" : { - "type" : "keyword" - }, - "Profiles" : { - "type" : "keyword" - }, - "Properties" : { - "type" : "keyword" - }, - "Protocol" : { - "type" : "keyword" - }, - "ProviderContextKey" : { - "type" : "keyword" - }, - "ProviderContextName" : { - "type" : "keyword" - }, - "ProviderContextType" : { - "type" : "keyword" - }, - "ProviderKey" : { - "type" : "keyword" - }, - "ProviderName" : { - "type" : "keyword" - }, - "ProviderType" : { - "type" : "keyword" - }, - "ProxyPolicyName" : { - "type" : "keyword" - }, - "PuaCount" : { - "type" : "keyword" - }, - "PuaPolicyId" : { - "type" : "keyword" - }, - "PublisherGuid" : { - "type" : "keyword" - }, - "PublisherID" : { - "type" : "keyword" - }, - "PublisherName" : { - "type" : "keyword" - }, - "QMFilterID" : { - "type" : "keyword" - }, - "QMLimit" : { - "type" : "keyword" - }, - "QuarantineHelpURL" : { - "type" : "keyword" - }, - "QuarantineSessionID" : { - "type" : "keyword" - }, - "QuarantineSessionIdentifier" : { - "type" : "keyword" - }, - "QuarantineState" : { - "type" : "keyword" - }, - "QuarantineSystemHealthResult" : { - "type" : "keyword" - }, - "QuickModeSaId" : { - "type" : "keyword" - }, - "Reason" : { - "type" : "keyword" - }, - "ReasonCode" : { - "type" : "keyword" - }, - "ReasonForRejection" : { - "type" : "keyword" - }, - "ReasonText" : { - "type" : "keyword" - }, - "RecoveryKeyId" : { - "type" : "keyword" - }, - "RecoveryReason" : { - "type" : "keyword" - }, - "RecoveryServer" : { - "type" : "keyword" - }, - "RelativeTargetName" : { - "type" : "keyword" - }, - "RemoteAddress" : { - "type" : "keyword" - }, - "RemoteAddressMask" : { - "type" : "keyword" - }, - "RemoteAdminEnabled" : { - "type" : "keyword" - }, - "RemoteEventLogging" : { - "type" : "keyword" - }, - "RemoteKeyModPort" : { - "type" : "keyword" - }, - "RemoteMMPrincipalName" : { - "type" : "keyword" - }, - "RemoteMachineID" : { - "type" : "keyword" - }, - "RemotePort" : { - "type" : "keyword" - }, - "RemotePrivateAddress" : { - "type" : "keyword" - }, - "RemoteTunnelEndpoint" : { - "type" : "keyword" - }, - "RemoteUserID" : { - "type" : "keyword" - }, - "RequestId" : { - "type" : "keyword" - }, - "RequestType" : { - "type" : "keyword" - }, - "Requester" : { - "type" : "keyword" - }, - "ResourceAttributes" : { - "type" : "keyword" - }, - "ResourceManager" : { - "type" : "keyword" - }, - "ResponderCookie" : { - "type" : "keyword" - }, - "RestrictedAdminMode" : { - "type" : "keyword" - }, - "RestrictedSidCount" : { - "type" : "keyword" - }, - "ReturnCode" : { - "type" : "keyword" - }, - "Role" : { - "type" : "keyword" - }, - "RuleAttr" : { - "type" : "keyword" - }, - "RuleId" : { - "type" : "keyword" - }, - "RuleName" : { - "type" : "keyword" - }, - "SPI" : { - "type" : "keyword" - }, - "SSID" : { - "type" : "keyword" - }, - "SamAccountName" : { - "type" : "keyword" - }, - "ScopeName" : { - "type" : "keyword" - }, - "ScriptPath" : { - "type" : "keyword" - }, - "SecurityDescriptor" : { - "type" : "keyword" - }, - "SecurityPackageName" : { - "type" : "keyword" - }, - "Service" : { - "type" : "keyword" - }, - "ServiceAccount" : { - "type" : "keyword" - }, - "ServiceFileName" : { - "type" : "keyword" - }, - "ServiceName" : { - "type" : "keyword" - }, - "ServicePrincipalNames" : { - "type" : "keyword" - }, - "ServiceSid" : { - "type" : "keyword" - }, - "ServiceStartType" : { - "type" : "keyword" - }, - "ServiceType" : { - "type" : "keyword" - }, - "SessionID" : { - "type" : "keyword" - }, - "SessionId" : { - "type" : "keyword" - }, - "SessionName" : { - "type" : "keyword" - }, - "SettingType" : { - "type" : "keyword" - }, - "SettingValue" : { - "type" : "keyword" - }, - "ShareLocalPath" : { - "type" : "keyword" - }, - "ShareName" : { - "type" : "keyword" - }, - "SidHistory" : { - "type" : "keyword" - }, - "SourceAddr" : { - "type" : "keyword" - }, - "SourceAddress" : { - "type" : "keyword" - }, - "SourceDRA" : { - "type" : "keyword" - }, - "SourceHandleId" : { - "type" : "keyword" - }, - "SourcePort" : { - "type" : "keyword" - }, - "SourceProcessId" : { - "type" : "keyword" - }, - "StartUSN" : { - "type" : "keyword" - }, - "State" : { - "type" : "keyword" - }, - "Status" : { - "type" : "keyword" - }, - "StatusCode" : { - "type" : "keyword" - }, - "StoreUrl" : { - "type" : "keyword" - }, - "SubLayerKey" : { - "type" : "keyword" - }, - "SubLayerName" : { - "type" : "keyword" - }, - "SubLayerType" : { - "type" : "keyword" - }, - "SubStatus" : { - "type" : "keyword" - }, - "SubcategoryGuid" : { - "type" : "keyword" - }, - "SubcategoryId" : { - "type" : "keyword" - }, - "Subject" : { - "type" : "keyword" - }, - "SubjectDomainName" : { - "type" : "keyword" - }, - "SubjectKeyIdentifier" : { - "type" : "keyword" - }, - "SubjectLogonId" : { - "type" : "keyword" - }, - "SubjectMachineName" : { - "type" : "keyword" - }, - "SubjectMachineSID" : { - "type" : "keyword" - }, - "SubjectUserDomainName" : { - "type" : "keyword" - }, - "SubjectUserName" : { - "type" : "keyword" - }, - "SubjectUserSid" : { - "type" : "keyword" - }, - "TargetDomainName" : { - "type" : "keyword" - }, - "TargetHandleId" : { - "type" : "keyword" - }, - "TargetInfo" : { - "type" : "keyword" - }, - "TargetLinkedLogonId" : { - "type" : "keyword" - }, - "TargetLogonGuid" : { - "type" : "keyword" - }, - "TargetLogonId" : { - "type" : "keyword" - }, - "TargetOutboundDomainName" : { - "type" : "keyword" - }, - "TargetOutboundUserName" : { - "type" : "keyword" - }, - "TargetProcessId" : { - "type" : "keyword" - }, - "TargetServerName" : { - "type" : "keyword" - }, - "TargetSid" : { - "type" : "keyword" - }, - "TargetUserName" : { - "type" : "keyword" - }, - "TargetUserSid" : { - "type" : "keyword" - }, - "TaskContent" : { - "type" : "keyword" - }, - "TaskContentNew" : { - "type" : "keyword" - }, - "TaskName" : { - "type" : "keyword" - }, - "TemplateContent" : { - "type" : "keyword" - }, - "TemplateDSObjectFQDN" : { - "type" : "keyword" - }, - "TemplateInternalName" : { - "type" : "keyword" - }, - "TemplateOID" : { - "type" : "keyword" - }, - "TemplateSchemaVersion" : { - "type" : "keyword" - }, - "TemplateVersion" : { - "type" : "keyword" - }, - "TestSigning" : { - "type" : "keyword" - }, - "TicketEncryptionType" : { - "type" : "keyword" - }, - "TicketOptions" : { - "type" : "keyword" - }, - "TokenElevationType" : { - "type" : "keyword" - }, - "TrafficSelectorId" : { - "type" : "keyword" - }, - "TransactionId" : { - "type" : "keyword" - }, - "TransmittedServices" : { - "type" : "keyword" - }, - "TransportFilterId" : { - "type" : "keyword" - }, - "TreeDelete" : { - "type" : "keyword" - }, - "TunnelId" : { - "type" : "keyword" - }, - "UserAccountControl" : { - "type" : "keyword" - }, - "UserName" : { - "type" : "keyword" - }, - "UserParameters" : { - "type" : "keyword" - }, - "UserPrincipalName" : { - "type" : "keyword" - }, - "UserSid" : { - "type" : "keyword" - }, - "UserWorkstations" : { - "type" : "keyword" - }, - "VirtualAccount" : { - "type" : "keyword" - }, - "VsmLaunchType" : { - "type" : "keyword" - }, - "Weight" : { - "type" : "keyword" - }, - "Workstation" : { - "type" : "keyword" - }, - "WorkstationName" : { - "type" : "keyword" - }, - "param1" : { - "type" : "keyword" - }, - "param2" : { - "type" : "keyword" - }, - "param3" : { - "type" : "keyword" - }, - "param4" : { - "type" : "keyword" - }, - "param5" : { - "type" : "keyword" - }, - "param6" : { - "type" : "keyword" - }, - "param7" : { - "type" : "keyword" - }, - "param8" : { - "type" : "keyword" - }, - "param9" : { - "type" : "keyword" - }, - "xml_name" : { - "type" : "keyword" - } - } - }, - "eventlog" : { - "properties" : { - "computer" : { - "type" : "keyword" - }, - "eventID" : { - "type" : "long" - }, - "event_recordID" : { - "type" : "long" - }, - "keywords" : { - "type" : "keyword" - }, - "log_name" : { - "type" : "keyword" - }, - "processID" : { - "type" : "long" - }, - "source" : { - "type" : "keyword" - }, - "task_category" : { - "type" : "keyword" - }, - "threadID" : { - "type" : "long" - } - } - }, - "hostIP" : { - "type" : "ip" - }, - "hostIP_geo" : { - "dynamic" : "true", - "properties" : { - "location" : { - "type" : "geo_point" - }, - "postal_code" : { - "type" : "keyword" - } - } - }, - "hostname" : { - "type" : "keyword" - }, - "message" : { - "type" : "text", - "norms" : false, - "fields" : { - "raw" : { - "type" : "keyword", - "ignore_above" : 512 - } - } - }, - "message_error" : { - "type" : "text", - "norms" : false, - "fields" : { - "raw" : { - "type" : "keyword", - "ignore_above" : 512 - } - } - }, - "meta" : { - "include_in_all" : false, - "properties" : { - "beat_type" : { - "type" : "keyword" - }, - "beat_version" : { - "type" : "keyword" - }, - "diff_indexer_shipper_ms" : { - "type" : "long" - }, - "diff_shipper_timestamp_ms" : { - "type" : "long" - }, - "kafka_topic" : { - "type" : "keyword" - }, - "processed_at_indexer" : { - "type" : "keyword" - }, - "processed_at_shipper" : { - "type" : "keyword" - }, - "received_at_indexer" : { - "type" : "date" - }, - "received_at_shipper" : { - "type" : "date" - }, - "received_from" : { - "type" : "keyword" - } - } - }, - "net" : { - "properties" : { - "dst_ip" : { - "type" : "ip" - }, - "dst_ip_geo" : { - "properties" : { - "location" : { - "type" : "geo_point" - } - } - }, - "dst_nat_ip" : { - "type" : "ip" - }, - "dst_nat_port" : { - "type" : "long" - }, - "dst_port" : { - "type" : "long" - }, - "src_domain" : { - "type" : "keyword" - }, - "src_ip" : { - "type" : "ip" - }, - "src_ip_geo" : { - "properties" : { - "location" : { - "type" : "geo_point" - } - } - }, - "src_nat_ip" : { - "type" : "ip" - }, - "src_nat_port" : { - "type" : "long" - }, - "src_port" : { - "type" : "long" - }, - "src_user" : { - "type" : "keyword" - }, - "src_user_department" : { - "type" : "keyword" - }, - "src_user_department_number" : { - "type" : "keyword" - }, - "src_user_display_name" : { - "type" : "keyword" - }, - "src_user_title" : { - "type" : "keyword" - } - } - }, - "severity" : { - "type" : "keyword" - }, - "tags" : { - "type" : "text", - "norms" : false, - "fields" : { - "raw" : { - "type" : "keyword", - "ignore_above" : 512 - } - } - }, - "type" : { - "type" : "keyword" - }, - "user" : { - "type" : "object" - } - } -} From 85b61cd8e41e0476a32241a18888258774f28baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 22 Nov 2024 10:46:11 +0100 Subject: [PATCH 04/24] Add basic search test --- .../oldrepos7x/OldMappingsIT.java | 239 ++++++++++-------- 1 file changed, 137 insertions(+), 102 deletions(-) diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java index e50e9ce590c86..e4fc8a0c8d963 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java @@ -35,6 +35,8 @@ public class OldMappingsIT extends ESRestTestCase { + private static final List indices = Arrays.asList("filebeat", "custom", "nested"); + public static TemporaryFolder repoDirectory = new TemporaryFolder(); public static ElasticsearchCluster currentCluster = ElasticsearchCluster.local() @@ -57,10 +59,11 @@ public class OldMappingsIT extends ESRestTestCase { @ClassRule public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(oldCluster).around(currentCluster); + private static boolean repoSetup = false; @Override protected String getTestRestCluster() { - return oldCluster.getHttpAddresses(); + return currentCluster.getHttpAddresses(); } private Request createIndex(String indexName, String file) throws IOException { @@ -71,9 +74,10 @@ private Request createIndex(String indexName, String file) throws IOException { .startObject() .startObject("settings") .field("index.number_of_shards", numberOfShards) + .field("index.number_of_replicas", 0) .endObject() .startObject("mappings"); - builder.rawValue(getClass().getResourceAsStream(file), XContentType.JSON); + builder.rawValue(OldMappingsIT.class.getResourceAsStream(file), XContentType.JSON); builder.endObject().endObject(); createIndex.setJsonEntity(Strings.toString(builder)); @@ -82,106 +86,108 @@ private Request createIndex(String indexName, String file) throws IOException { @Before public void setupIndex() throws IOException { - // String repoLocation = PathUtils.get(System.getProperty("tests.repo.location")) - // .resolve(RandomizedTest.getContext().getTargetClass().getName()) - // .toString(); - - String repoLocation = repoDirectory.getRoot().getPath(); - - String repoName = "old_mappings_repo"; - String snapshotName = "snap"; - List indices; - indices = Arrays.asList("filebeat", "custom", "nested"); - - List oldClusterHosts = parseClusterHosts(oldCluster.getHttpAddresses()); - List clusterHosts = parseClusterHosts(currentCluster.getHttpAddresses()); - // try (RestClient oldEsClient = client()) { - try ( - RestClient oldEsClient = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build(); - RestClient esClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()])).build(); - ) { - assertOK(oldEsClient.performRequest(createIndex("filebeat", "filebeat.json"))); - - assertOK(oldEsClient.performRequest(createIndex("custom", "custom.json"))); - assertOK(oldEsClient.performRequest(createIndex("nested", "nested.json"))); - - Request doc1 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "1"); - doc1.addParameter("refresh", "true"); - XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() - .startObject() - .startObject("apache2") - .startObject("access") - .field("url", "myurl1") - .field("agent", "agent1") - .endObject() - .endObject() - .endObject(); - doc1.setJsonEntity(Strings.toString(bodyDoc1)); - assertOK(oldEsClient.performRequest(doc1)); - - Request doc2 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "2"); - doc2.addParameter("refresh", "true"); - XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() - .startObject() - .startObject("apache2") - .startObject("access") - .field("url", "myurl2") - .field("agent", "agent2 agent2") - .endObject() - .endObject() - .field("completion", "some_value") - .endObject(); - doc2.setJsonEntity(Strings.toString(bodyDoc2)); - assertOK(oldEsClient.performRequest(doc2)); - - Request doc3 = new Request("PUT", "/" + "nested" + "/" + "_doc" + "/" + "1"); - doc3.addParameter("refresh", "true"); - XContentBuilder bodyDoc3 = XContentFactory.jsonBuilder() - .startObject() - .field("group", "fans") - .startArray("user") - .startObject() - .field("first", "John") - .field("last", "Smith") - .endObject() - .startObject() - .field("first", "Alice") - .field("last", "White") - .endObject() - .endArray() - .endObject(); - doc3.setJsonEntity(Strings.toString(bodyDoc3)); - assertOK(oldEsClient.performRequest(doc3)); - - Request getSettingsRequest = new Request("GET", "/_cluster/settings?include_defaults=true"); - Map response = entityAsMap(oldEsClient.performRequest(getSettingsRequest)); - assertEquals(repoLocation, ((List) (XContentMapValues.extractValue("defaults.path.repo", response))).get(0)); - - // register repo on old ES and take snapshot - Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest.setJsonEntity(Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(oldEsClient.performRequest(createRepoRequest)); - - Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); - createSnapshotRequest.addParameter("wait_for_completion", "true"); - createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); - assertOK(oldEsClient.performRequest(createSnapshotRequest)); - // } - - // register repo on new ES and restore snapshot - Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest2.setJsonEntity(Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(esClient.performRequest(createRepoRequest2)); - - final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); - createRestoreRequest.addParameter("wait_for_completion", "true"); - createRestoreRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); - createRestoreRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); - assertOK(esClient.performRequest(createRestoreRequest)); + if (repoSetup == false) { + // String repoLocation = PathUtils.get(System.getProperty("tests.repo.location")) + // .resolve(RandomizedTest.getContext().getTargetClass().getName()) + // .toString(); + + String repoLocation = repoDirectory.getRoot().getPath(); + + String repoName = "old_mappings_repo"; + String snapshotName = "snap"; + + List oldClusterHosts = parseClusterHosts(oldCluster.getHttpAddresses()); + List clusterHosts = parseClusterHosts(currentCluster.getHttpAddresses()); + // try (RestClient oldEsClient = client()) { + try ( + RestClient oldEsClient = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build(); + RestClient esClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()])).build(); + ) { + assertOK(oldEsClient.performRequest(createIndex("filebeat", "filebeat.json"))); + + assertOK(oldEsClient.performRequest(createIndex("custom", "custom.json"))); + assertOK(oldEsClient.performRequest(createIndex("nested", "nested.json"))); + + Request doc1 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "1"); + doc1.addParameter("refresh", "true"); + XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() + .startObject() + .startObject("apache2") + .startObject("access") + .field("url", "myurl1") + .field("agent", "agent1") + .endObject() + .endObject() + .endObject(); + doc1.setJsonEntity(Strings.toString(bodyDoc1)); + assertOK(oldEsClient.performRequest(doc1)); + + Request doc2 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "2"); + doc2.addParameter("refresh", "true"); + XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() + .startObject() + .startObject("apache2") + .startObject("access") + .field("url", "myurl2") + .field("agent", "agent2 agent2") + .endObject() + .endObject() + .field("completion", "some_value") + .endObject(); + doc2.setJsonEntity(Strings.toString(bodyDoc2)); + assertOK(oldEsClient.performRequest(doc2)); + + Request doc3 = new Request("PUT", "/" + "nested" + "/" + "_doc" + "/" + "1"); + doc3.addParameter("refresh", "true"); + XContentBuilder bodyDoc3 = XContentFactory.jsonBuilder() + .startObject() + .field("group", "fans") + .startArray("user") + .startObject() + .field("first", "John") + .field("last", "Smith") + .endObject() + .startObject() + .field("first", "Alice") + .field("last", "White") + .endObject() + .endArray() + .endObject(); + doc3.setJsonEntity(Strings.toString(bodyDoc3)); + assertOK(oldEsClient.performRequest(doc3)); + + Request getSettingsRequest = new Request("GET", "/_cluster/settings?include_defaults=true"); + Map response = entityAsMap(oldEsClient.performRequest(getSettingsRequest)); + assertEquals(repoLocation, ((List) (XContentMapValues.extractValue("defaults.path.repo", response))).get(0)); + + // register repo on old ES and take snapshot + Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(oldEsClient.performRequest(createRepoRequest)); + + Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); + assertOK(oldEsClient.performRequest(createSnapshotRequest)); + // } + + // register repo on new ES and restore snapshot + Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest2.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(esClient.performRequest(createRepoRequest2)); + + final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); + createRestoreRequest.addParameter("wait_for_completion", "true"); + createRestoreRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); + createRestoreRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + assertOK(esClient.performRequest(createRestoreRequest)); + + repoSetup = true; + } } } @@ -191,4 +197,33 @@ public void testMappingOk() throws IOException { // assertNotNull(XContentMapValues.extractValue(mapping, "filebeat", "mappings", "properties", "apache2")); } + public void testBasicSearchOk() throws IOException { + System.out.println("TestBasicSearchOk"); + List clusterHosts = parseClusterHosts(currentCluster.getHttpAddresses()); + try (RestClient esClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()])).build();) { + for (String index : indices) { + ensureGreen(esClient, index); + } + Request searchRequest = new Request("POST", "/custom/_search"); + Map searchResponse = entityAsMap(esClient.performRequest(searchRequest)); + assertEquals(2, XContentMapValues.extractValue(searchResponse, "hits", "total", "value")); + } + } + + protected boolean resetFeatureStates() { + return false; + } + + protected boolean preserveIndicesUponCompletion() { + return true; + } + + protected void wipeSnapshots() throws IOException { + // we want to preserve snapshots between individual tests + } + + @Override + protected String getEnsureGreenTimeout() { + return "5s"; + } } From 93b95fb7c265cdf4c0306f15929a237912ce52f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 25 Nov 2024 15:46:43 +0100 Subject: [PATCH 05/24] Rework OldMappingIT --- .../test/rest/ESRestTestCase.java | 4 +- .../oldrepos7x/OldMappingsIT.java | 413 +++-- .../elasticsearch/oldrepos7x/winlogbeat.json | 1610 ----------------- 3 files changed, 289 insertions(+), 1738 deletions(-) delete mode 100644 x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/winlogbeat.json diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index dd08107bd67fb..c0977cd44ac7f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -10,7 +10,6 @@ package org.elasticsearch.test.rest; import io.netty.handler.codec.http.HttpMethod; - import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -88,6 +87,7 @@ import org.junit.AfterClass; import org.junit.Before; +import javax.net.ssl.SSLContext; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -126,8 +126,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import javax.net.ssl.SSLContext; - import static java.util.Collections.sort; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java index e4fc8a0c8d963..ca1c296bd59be 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java @@ -10,6 +10,7 @@ import org.apache.http.HttpHost; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.common.Strings; @@ -22,17 +23,24 @@ import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import static java.util.Collections.unmodifiableList; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; + public class OldMappingsIT extends ESRestTestCase { private static final List indices = Arrays.asList("filebeat", "custom", "nested"); @@ -59,14 +67,18 @@ public class OldMappingsIT extends ESRestTestCase { @ClassRule public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(oldCluster).around(currentCluster); - private static boolean repoSetup = false; + + private static boolean repoRestored = false; + private static String repoLocation; + private static String repoName; + private static String snapshotName; @Override protected String getTestRestCluster() { return currentCluster.getHttpAddresses(); } - private Request createIndex(String indexName, String file) throws IOException { + private static Request createIndex(String indexName, String file) throws IOException { Request createIndex = new Request("PUT", "/" + indexName); int numberOfShards = randomIntBetween(1, 3); @@ -84,130 +96,286 @@ private Request createIndex(String indexName, String file) throws IOException { return createIndex; } - @Before - public void setupIndex() throws IOException { - if (repoSetup == false) { - // String repoLocation = PathUtils.get(System.getProperty("tests.repo.location")) - // .resolve(RandomizedTest.getContext().getTargetClass().getName()) - // .toString(); - - String repoLocation = repoDirectory.getRoot().getPath(); - - String repoName = "old_mappings_repo"; - String snapshotName = "snap"; - - List oldClusterHosts = parseClusterHosts(oldCluster.getHttpAddresses()); - List clusterHosts = parseClusterHosts(currentCluster.getHttpAddresses()); - // try (RestClient oldEsClient = client()) { - try ( - RestClient oldEsClient = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build(); - RestClient esClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()])).build(); - ) { - assertOK(oldEsClient.performRequest(createIndex("filebeat", "filebeat.json"))); - - assertOK(oldEsClient.performRequest(createIndex("custom", "custom.json"))); - assertOK(oldEsClient.performRequest(createIndex("nested", "nested.json"))); - - Request doc1 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "1"); - doc1.addParameter("refresh", "true"); - XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() - .startObject() - .startObject("apache2") - .startObject("access") - .field("url", "myurl1") - .field("agent", "agent1") - .endObject() - .endObject() - .endObject(); - doc1.setJsonEntity(Strings.toString(bodyDoc1)); - assertOK(oldEsClient.performRequest(doc1)); - - Request doc2 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "2"); - doc2.addParameter("refresh", "true"); - XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() - .startObject() - .startObject("apache2") - .startObject("access") - .field("url", "myurl2") - .field("agent", "agent2 agent2") - .endObject() - .endObject() - .field("completion", "some_value") - .endObject(); - doc2.setJsonEntity(Strings.toString(bodyDoc2)); - assertOK(oldEsClient.performRequest(doc2)); - - Request doc3 = new Request("PUT", "/" + "nested" + "/" + "_doc" + "/" + "1"); - doc3.addParameter("refresh", "true"); - XContentBuilder bodyDoc3 = XContentFactory.jsonBuilder() - .startObject() - .field("group", "fans") - .startArray("user") - .startObject() - .field("first", "John") - .field("last", "Smith") - .endObject() - .startObject() - .field("first", "Alice") - .field("last", "White") - .endObject() - .endArray() - .endObject(); - doc3.setJsonEntity(Strings.toString(bodyDoc3)); - assertOK(oldEsClient.performRequest(doc3)); - - Request getSettingsRequest = new Request("GET", "/_cluster/settings?include_defaults=true"); - Map response = entityAsMap(oldEsClient.performRequest(getSettingsRequest)); - assertEquals(repoLocation, ((List) (XContentMapValues.extractValue("defaults.path.repo", response))).get(0)); - - // register repo on old ES and take snapshot - Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest.setJsonEntity(Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(oldEsClient.performRequest(createRepoRequest)); - - Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); - createSnapshotRequest.addParameter("wait_for_completion", "true"); - createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); - assertOK(oldEsClient.performRequest(createSnapshotRequest)); - // } - - // register repo on new ES and restore snapshot - Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest2.setJsonEntity(Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(esClient.performRequest(createRepoRequest2)); - - final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); - createRestoreRequest.addParameter("wait_for_completion", "true"); - createRestoreRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); - createRestoreRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); - assertOK(esClient.performRequest(createRestoreRequest)); - - repoSetup = true; - } + @BeforeClass + public static void setupOldRepo() throws IOException { + repoLocation = repoDirectory.getRoot().getPath(); + + repoName = "old_mappings_repo"; + snapshotName = "snap"; + + List oldClusterHosts = parseOldClusterHosts(oldCluster.getHttpAddresses()); + try (RestClient oldEsClient = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build();) { + assertOK(oldEsClient.performRequest(createIndex("filebeat", "filebeat.json"))); + + assertOK(oldEsClient.performRequest(createIndex("custom", "custom.json"))); + assertOK(oldEsClient.performRequest(createIndex("nested", "nested.json"))); + + Request doc1 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "1"); + doc1.addParameter("refresh", "true"); + XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() + .startObject() + .startObject("apache2") + .startObject("access") + .field("url", "myurl1") + .field("agent", "agent1") + .endObject() + .endObject() + .endObject(); + doc1.setJsonEntity(Strings.toString(bodyDoc1)); + assertOK(oldEsClient.performRequest(doc1)); + + Request doc2 = new Request("PUT", "/" + "custom" + "/" + "_doc" + "/" + "2"); + doc2.addParameter("refresh", "true"); + XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() + .startObject() + .startObject("apache2") + .startObject("access") + .field("url", "myurl2") + .field("agent", "agent2 agent2") + .endObject() + .endObject() + .field("completion", "some_value") + .endObject(); + doc2.setJsonEntity(Strings.toString(bodyDoc2)); + assertOK(oldEsClient.performRequest(doc2)); + + Request doc3 = new Request("PUT", "/" + "nested" + "/" + "_doc" + "/" + "1"); + doc3.addParameter("refresh", "true"); + XContentBuilder bodyDoc3 = XContentFactory.jsonBuilder() + .startObject() + .field("group", "fans") + .startArray("user") + .startObject() + .field("first", "John") + .field("last", "Smith") + .endObject() + .startObject() + .field("first", "Alice") + .field("last", "White") + .endObject() + .endArray() + .endObject(); + doc3.setJsonEntity(Strings.toString(bodyDoc3)); + assertOK(oldEsClient.performRequest(doc3)); + + Request getSettingsRequest = new Request("GET", "/_cluster/settings?include_defaults=true"); + Map response = entityAsMap(oldEsClient.performRequest(getSettingsRequest)); + assertEquals(repoLocation, ((List) (XContentMapValues.extractValue("defaults.path.repo", response))).get(0)); + + // register repo on old ES and take snapshot + Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(oldEsClient.performRequest(createRepoRequest)); + + Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); + assertOK(oldEsClient.performRequest(createSnapshotRequest)); + } } - public void testMappingOk() throws IOException { - // Request mappingRequest = new Request("GET", "/" + "filebeat" + "/_mapping"); - // Map mapping = entityAsMap(client().performRequest(mappingRequest)); - // assertNotNull(XContentMapValues.extractValue(mapping, "filebeat", "mappings", "properties", "apache2")); + @Before + public void registerAndRestoreRepo() throws IOException { + if (repoRestored == false) { + // register repo on new ES and restore snapshot + Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest2.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(client().performRequest(createRepoRequest2)); + + final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); + createRestoreRequest.addParameter("wait_for_completion", "true"); + createRestoreRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); + createRestoreRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + assertOK(client().performRequest(createRestoreRequest)); + + repoRestored = true; + } } - public void testBasicSearchOk() throws IOException { - System.out.println("TestBasicSearchOk"); - List clusterHosts = parseClusterHosts(currentCluster.getHttpAddresses()); - try (RestClient esClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()])).build();) { - for (String index : indices) { - ensureGreen(esClient, index); + private static List parseOldClusterHosts(String hostsString) { + String[] stringUrls = hostsString.split(","); + List hosts = new ArrayList<>(stringUrls.length); + for (String stringUrl : stringUrls) { + int portSeparator = stringUrl.lastIndexOf(':'); + if (portSeparator < 0) { + throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]"); } - Request searchRequest = new Request("POST", "/custom/_search"); - Map searchResponse = entityAsMap(esClient.performRequest(searchRequest)); - assertEquals(2, XContentMapValues.extractValue(searchResponse, "hits", "total", "value")); + String host = stringUrl.substring(0, portSeparator); + int port = Integer.valueOf(stringUrl.substring(portSeparator + 1)); + hosts.add(new HttpHost(host, port)); } + return unmodifiableList(hosts); + } + + public void testFileBeatApache2MappingOk() throws IOException { + Request mappingRequest = new Request("GET", "/" + "filebeat" + "/_mapping"); + Map mapping = entityAsMap(client().performRequest(mappingRequest)); + assertNotNull(XContentMapValues.extractValue(mapping, "filebeat", "mappings", "properties", "apache2")); + } + + public void testSearchKeyword() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("match") + .startObject("apache2.access.url") + .field("query", "myurl2") + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + Map response = entityAsMap(client().performRequest(search)); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + } + + public void testSearchOnPlaceHolderField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("match") + .startObject("completion") + .field("query", "some-agent") + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + ResponseException re = expectThrows(ResponseException.class, () -> entityAsMap(client().performRequest(search))); + assertThat( + re.getMessage(), + containsString("Field [completion] of type [completion] in legacy index does not support match queries") + ); + } + + public void testAggregationOnPlaceholderField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("aggs") + .startObject("agents") + .startObject("terms") + .field("field", "completion") + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + ResponseException re = expectThrows(ResponseException.class, () -> entityAsMap(client().performRequest(search))); + assertThat(re.getMessage(), containsString("can't run aggregation or sorts on field type completion of legacy index")); + } + + public void testConstantScoringOnTextField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("match") + .startObject("apache2.access.agent") + .field("query", "agent2") + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + Map response = entityAsMap(client().performRequest(search)); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + @SuppressWarnings("unchecked") + Map hit = (Map) hits.get(0); + assertThat(hit, hasKey("_score")); + assertEquals(1.0d, (double) hit.get("_score"), 0.01d); + } + + public void testFieldsExistQueryOnTextField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("exists") + .field("field", "apache2.access.agent") + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + Map response = entityAsMap(client().performRequest(search)); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(2)); + } + + public void testSearchFieldsOnPlaceholderField() throws IOException { + Request search = new Request("POST", "/" + "custom" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("match") + .startObject("apache2.access.url") + .field("query", "myurl2") + .endObject() + .endObject() + .endObject() + .startArray("fields") + .value("completion") + .endArray() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + Map response = entityAsMap(client().performRequest(search)); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + logger.info(hits); + Map fields = (Map) (XContentMapValues.extractValue("fields", (Map) hits.get(0))); + assertEquals(List.of("some_value"), fields.get("completion")); + } + + public void testNestedDocuments() throws IOException { + Request search = new Request("POST", "/" + "nested" + "/_search"); + Map response = entityAsMap(client().performRequest(search)); + logger.info(response); + List hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + Map source = (Map) (XContentMapValues.extractValue("_source", (Map) hits.get(0))); + assertEquals("fans", source.get("group")); + + search = new Request("POST", "/" + "nested" + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("query") + .startObject("nested") + .field("path", "user") + .startObject("query") + .startObject("bool") + .startArray("must") + .startObject() + .startObject("match") + .field("user.first", "Alice") + .endObject() + .endObject() + .startObject() + .startObject("match") + .field("user.last", "White") + .endObject() + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + search.setJsonEntity(Strings.toString(query)); + response = entityAsMap(client().performRequest(search)); + logger.info(response); + hits = (List) (XContentMapValues.extractValue("hits.hits", response)); + assertThat(hits, hasSize(1)); + source = (Map) (XContentMapValues.extractValue("_source", (Map) hits.get(0))); + assertEquals("fans", source.get("group")); } protected boolean resetFeatureStates() { @@ -219,11 +387,6 @@ protected boolean preserveIndicesUponCompletion() { } protected void wipeSnapshots() throws IOException { - // we want to preserve snapshots between individual tests - } - - @Override - protected String getEnsureGreenTimeout() { - return "5s"; + // we want to keep snapshots between individual tests } } diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/winlogbeat.json b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/winlogbeat.json deleted file mode 100644 index 8fcd4048bf113..0000000000000 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/winlogbeat.json +++ /dev/null @@ -1,1610 +0,0 @@ -"_all" : { - "enabled" : true, - "norms" : false -}, -"dynamic_templates" : [ - { - "message_field" : { - "path_match" : "*message*", - "match_mapping_type" : "string", - "mapping" : { - "fields" : { - "raw" : { - "ignore_above" : 512, - "type" : "keyword" - } - }, - "norms" : false, - "type" : "text" - } - } - }, - { - "tags_field" : { - "path_match" : "*tags*", - "match_mapping_type" : "string", - "mapping" : { - "fields" : { - "raw" : { - "ignore_above" : 512, - "type" : "keyword" - } - }, - "norms" : false, - "type" : "text" - } - } - }, - { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "norms" : false, - "type" : "keyword" - } - } - } -], -"properties" : { - "@timestamp" : { - "type" : "date", - "include_in_all" : false - }, - "@version" : { - "type" : "keyword", - "include_in_all" : false - }, - "hostIP" : { - "type" : "ip" - }, - "hostIP_geo" : { - "dynamic" : "true", - "properties" : { - "location" : { - "type" : "geo_point" - }, - "postal_code" : { - "type" : "keyword" - } - } - }, - "hostname" : { - "type" : "keyword" - }, - "meta" : { - "include_in_all" : false, - "properties" : { - "processed_at_indexer" : { - "type" : "keyword" - }, - "processed_at_shipper" : { - "type" : "keyword" - }, - "received_from" : { - "type" : "keyword" - } - } - }, - "net" : { - "properties" : { - "dst_ip" : { - "type" : "ip" - }, - "dst_ip_geo" : { - "properties" : { - "location" : { - "type" : "geo_point" - } - } - }, - "dst_nat_ip" : { - "type" : "ip" - }, - "dst_nat_port" : { - "type" : "long" - }, - "dst_port" : { - "type" : "long" - }, - "src_ip" : { - "type" : "ip" - }, - "src_ip_geo" : { - "properties" : { - "location" : { - "type" : "geo_point" - } - } - }, - "src_nat_ip" : { - "type" : "ip" - }, - "src_nat_port" : { - "type" : "long" - }, - "src_port" : { - "type" : "long" - } - } - }, - "type" : { - "type" : "keyword" - } -} -}, -"wineventlog" : { -"_all" : { - "enabled" : true, - "norms" : false -}, -"dynamic_templates" : [ - { - "message_field" : { - "path_match" : "*message*", - "match_mapping_type" : "string", - "mapping" : { - "fields" : { - "raw" : { - "ignore_above" : 512, - "type" : "keyword" - } - }, - "norms" : false, - "type" : "text" - } - } - }, - { - "tags_field" : { - "path_match" : "*tags*", - "match_mapping_type" : "string", - "mapping" : { - "fields" : { - "raw" : { - "ignore_above" : 512, - "type" : "keyword" - } - }, - "norms" : false, - "type" : "text" - } - } - }, - { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "norms" : false, - "type" : "keyword" - } - } - } -], -"properties" : { - "@timestamp" : { - "type" : "date", - "include_in_all" : false - }, - "@version" : { - "type" : "keyword", - "include_in_all" : false - }, - "beat" : { - "type" : "object" - }, - "eventdata" : { - "properties" : { - "AccessGranted" : { - "type" : "keyword" - }, - "AccessList" : { - "type" : "keyword" - }, - "AccessMask" : { - "type" : "keyword" - }, - "AccessReason" : { - "type" : "keyword" - }, - "AccessRemoved" : { - "type" : "keyword" - }, - "AccountDomain" : { - "type" : "keyword" - }, - "AccountExpires" : { - "type" : "keyword" - }, - "AccountName" : { - "type" : "keyword" - }, - "AccountSessionIdentifier" : { - "type" : "keyword" - }, - "Action" : { - "type" : "keyword" - }, - "ActiveProfile" : { - "type" : "keyword" - }, - "AdditionalInfo" : { - "type" : "keyword" - }, - "AdditionalInfo2" : { - "type" : "keyword" - }, - "AdvancedOptions" : { - "type" : "keyword" - }, - "AhAuthType" : { - "type" : "keyword" - }, - "AlgorithmName" : { - "type" : "keyword" - }, - "AllowedToDelegateTo" : { - "type" : "keyword" - }, - "AppCorrelationID" : { - "type" : "keyword" - }, - "AppInstance" : { - "type" : "keyword" - }, - "AppName" : { - "type" : "keyword" - }, - "Application" : { - "type" : "keyword" - }, - "AttributeLDAPDisplayName" : { - "type" : "keyword" - }, - "AttributeSyntaxOID" : { - "type" : "keyword" - }, - "AttributeValue" : { - "type" : "keyword" - }, - "Attributes" : { - "type" : "keyword" - }, - "AuditPolicyChanges" : { - "type" : "keyword" - }, - "AuditSourceName" : { - "type" : "keyword" - }, - "AuditsDiscarded" : { - "type" : "keyword" - }, - "AuthenticationPackage" : { - "type" : "keyword" - }, - "AuthenticationPackageName" : { - "type" : "keyword" - }, - "AuthenticationProvider" : { - "type" : "keyword" - }, - "AuthenticationServer" : { - "type" : "keyword" - }, - "AuthenticationType" : { - "type" : "keyword" - }, - "BackupPath" : { - "type" : "keyword" - }, - "BackupType" : { - "type" : "keyword" - }, - "CalledStationID" : { - "type" : "keyword" - }, - "CallerProcessId" : { - "type" : "keyword" - }, - "CallerProcessName" : { - "type" : "keyword" - }, - "CallingStationID" : { - "type" : "keyword" - }, - "CalloutId" : { - "type" : "keyword" - }, - "CalloutKey" : { - "type" : "keyword" - }, - "CalloutName" : { - "type" : "keyword" - }, - "CalloutType" : { - "type" : "keyword" - }, - "Categories" : { - "type" : "keyword" - }, - "CategoryId" : { - "type" : "keyword" - }, - "ChangeType" : { - "type" : "keyword" - }, - "Channel" : { - "type" : "keyword" - }, - "CipherType" : { - "type" : "keyword" - }, - "ClientAddress" : { - "type" : "keyword" - }, - "ClientDomain" : { - "type" : "keyword" - }, - "ClientIPAddress" : { - "type" : "keyword" - }, - "ClientLogonId" : { - "type" : "keyword" - }, - "ClientName" : { - "type" : "keyword" - }, - "ClientUserName" : { - "type" : "keyword" - }, - "ComputerAccountChange" : { - "type" : "keyword" - }, - "Conditions" : { - "type" : "keyword" - }, - "ConfigAccessPolicy" : { - "type" : "keyword" - }, - "DCDNSName" : { - "type" : "keyword" - }, - "DHGroup" : { - "type" : "keyword" - }, - "DSName" : { - "type" : "keyword" - }, - "DSType" : { - "type" : "keyword" - }, - "DestAddress" : { - "type" : "keyword" - }, - "DestPort" : { - "type" : "keyword" - }, - "DestinationDRA" : { - "type" : "keyword" - }, - "Direction" : { - "type" : "keyword" - }, - "DisableIntegrityChecks" : { - "type" : "keyword" - }, - "DisabledPrivilegeList" : { - "type" : "keyword" - }, - "DisplayName" : { - "type" : "keyword" - }, - "Disposition" : { - "type" : "keyword" - }, - "DnsHostName" : { - "type" : "keyword" - }, - "DomainBehaviorVersion" : { - "type" : "keyword" - }, - "DomainName" : { - "type" : "keyword" - }, - "DomainPolicyChanged" : { - "type" : "keyword" - }, - "DomainSid" : { - "type" : "keyword" - }, - "Dummy" : { - "type" : "keyword" - }, - "EAPErrorCode" : { - "type" : "keyword" - }, - "EAPReasonCode" : { - "type" : "keyword" - }, - "EAPType" : { - "type" : "keyword" - }, - "ElevatedToken" : { - "type" : "keyword" - }, - "EnabledPrivilegeList" : { - "type" : "keyword" - }, - "EndUSN" : { - "type" : "keyword" - }, - "ErrorCode" : { - "type" : "keyword" - }, - "EspAuthType" : { - "type" : "keyword" - }, - "EventCountTotal" : { - "type" : "keyword" - }, - "EventID" : { - "type" : "keyword" - }, - "EventIdx" : { - "type" : "keyword" - }, - "EventSourceId" : { - "type" : "keyword" - }, - "ExtendedQuarantineState" : { - "type" : "keyword" - }, - "FailureDescription" : { - "type" : "keyword" - }, - "FailureId" : { - "type" : "keyword" - }, - "FailurePoint" : { - "type" : "keyword" - }, - "FailureReason" : { - "type" : "keyword" - }, - "FileName" : { - "type" : "keyword" - }, - "FilterId" : { - "type" : "keyword" - }, - "FilterKey" : { - "type" : "keyword" - }, - "FilterName" : { - "type" : "keyword" - }, - "FilterRTID" : { - "type" : "keyword" - }, - "FilterType" : { - "type" : "keyword" - }, - "FlightSigning" : { - "type" : "keyword" - }, - "ForceLogoff" : { - "type" : "keyword" - }, - "FullyQualifiedSubjectMachineName" : { - "type" : "keyword" - }, - "FullyQualifiedSubjectUserName" : { - "type" : "keyword" - }, - "GPOList" : { - "type" : "keyword" - }, - "Group" : { - "type" : "keyword" - }, - "GroupMembership" : { - "type" : "keyword" - }, - "GroupPolicyApplied" : { - "type" : "keyword" - }, - "HandleId" : { - "type" : "keyword" - }, - "HomeDirectory" : { - "type" : "keyword" - }, - "HomePath" : { - "type" : "keyword" - }, - "HypervisorDebug" : { - "type" : "keyword" - }, - "HypervisorLaunchType" : { - "type" : "keyword" - }, - "HypervisorLoadOptions" : { - "type" : "keyword" - }, - "Identity" : { - "type" : "keyword" - }, - "ImpersonationLevel" : { - "type" : "keyword" - }, - "InboundSpi" : { - "type" : "keyword" - }, - "InitiatorCookie" : { - "type" : "keyword" - }, - "InterfaceName" : { - "type" : "keyword" - }, - "IntfGuid" : { - "type" : "keyword" - }, - "IpAddress" : { - "type" : "keyword" - }, - "IpPort" : { - "type" : "keyword" - }, - "IpProtocol" : { - "type" : "keyword" - }, - "KernelDebug" : { - "type" : "keyword" - }, - "KeyFilePath" : { - "type" : "keyword" - }, - "KeyLength" : { - "type" : "keyword" - }, - "KeyModName" : { - "type" : "keyword" - }, - "KeyName" : { - "type" : "keyword" - }, - "KeyType" : { - "type" : "keyword" - }, - "KeyingModuleName" : { - "type" : "keyword" - }, - "LayerId" : { - "type" : "keyword" - }, - "LayerKey" : { - "type" : "keyword" - }, - "LayerName" : { - "type" : "keyword" - }, - "LayerRTID" : { - "type" : "keyword" - }, - "LifetimeKilobytes" : { - "type" : "keyword" - }, - "LifetimePackets" : { - "type" : "keyword" - }, - "LifetimeSeconds" : { - "type" : "keyword" - }, - "LinkName" : { - "type" : "keyword" - }, - "LmPackageName" : { - "type" : "keyword" - }, - "LoadOptions" : { - "type" : "keyword" - }, - "LocalAddress" : { - "type" : "keyword" - }, - "LocalAddressMask" : { - "type" : "keyword" - }, - "LocalKeyModPort" : { - "type" : "keyword" - }, - "LocalMMPrincipalName" : { - "type" : "keyword" - }, - "LocalMac" : { - "type" : "keyword" - }, - "LocalPort" : { - "type" : "keyword" - }, - "LocalTunnelEndpoint" : { - "type" : "keyword" - }, - "LockoutDuration" : { - "type" : "keyword" - }, - "LockoutObservationWindow" : { - "type" : "keyword" - }, - "LockoutThreshold" : { - "type" : "keyword" - }, - "LogDroppedPacketsEnabled" : { - "type" : "keyword" - }, - "LogSuccessfulConnectionsEnabled" : { - "type" : "keyword" - }, - "LoggingResult" : { - "type" : "keyword" - }, - "LogonGuid" : { - "type" : "keyword" - }, - "LogonHours" : { - "type" : "keyword" - }, - "LogonID" : { - "type" : "keyword" - }, - "LogonProcessName" : { - "type" : "keyword" - }, - "LogonType" : { - "type" : "keyword" - }, - "LogonType_Description" : { - "type" : "keyword" - }, - "MMAuthMethod" : { - "type" : "keyword" - }, - "MMCipherAlg" : { - "type" : "keyword" - }, - "MMFilterID" : { - "type" : "keyword" - }, - "MMImpersonationState" : { - "type" : "keyword" - }, - "MMIntegrityAlg" : { - "type" : "keyword" - }, - "MMLifetime" : { - "type" : "keyword" - }, - "MMSAID" : { - "type" : "keyword" - }, - "MachineAccountQuota" : { - "type" : "keyword" - }, - "MachineInventory" : { - "type" : "keyword" - }, - "MainModeSaId" : { - "type" : "keyword" - }, - "MandatoryLabel" : { - "type" : "keyword" - }, - "MappedName" : { - "type" : "keyword" - }, - "MappingBy" : { - "type" : "keyword" - }, - "MasterKeyId" : { - "type" : "keyword" - }, - "MaxPasswordAge" : { - "type" : "keyword" - }, - "MemberName" : { - "type" : "keyword" - }, - "MemberSid" : { - "type" : "keyword" - }, - "Message" : { - "type" : "keyword" - }, - "MessageID" : { - "type" : "keyword" - }, - "MinPasswordAge" : { - "type" : "keyword" - }, - "MinPasswordLength" : { - "type" : "keyword" - }, - "MixedDomainMode" : { - "type" : "keyword" - }, - "Mode" : { - "type" : "keyword" - }, - "ModifiedObjectProperties" : { - "type" : "keyword" - }, - "Module" : { - "type" : "keyword" - }, - "MulticastFlowsEnabled" : { - "type" : "keyword" - }, - "NASIPv4Address" : { - "type" : "keyword" - }, - "NASIPv6Address" : { - "type" : "keyword" - }, - "NASIdentifier" : { - "type" : "keyword" - }, - "NASPort" : { - "type" : "keyword" - }, - "NASPortType" : { - "type" : "keyword" - }, - "NamingContext" : { - "type" : "keyword" - }, - "NetworkPolicyName" : { - "type" : "keyword" - }, - "NewMaxUsers" : { - "type" : "keyword" - }, - "NewProcessId" : { - "type" : "keyword" - }, - "NewProcessName" : { - "type" : "keyword" - }, - "NewRemark" : { - "type" : "keyword" - }, - "NewSD" : { - "type" : "keyword" - }, - "NewSd" : { - "type" : "keyword" - }, - "NewShareFlags" : { - "type" : "keyword" - }, - "NewState" : { - "type" : "keyword" - }, - "NewTargetUserName" : { - "type" : "keyword" - }, - "NewTime" : { - "type" : "date" - }, - "NewUacValue" : { - "type" : "keyword" - }, - "NewValue" : { - "type" : "keyword" - }, - "NewValueType" : { - "type" : "keyword" - }, - "NotificationPackageName" : { - "type" : "keyword" - }, - "ObjectClass" : { - "type" : "keyword" - }, - "ObjectCollectionName" : { - "type" : "keyword" - }, - "ObjectDN" : { - "type" : "keyword" - }, - "ObjectGUID" : { - "type" : "keyword" - }, - "ObjectIdentifyingProperties" : { - "type" : "keyword" - }, - "ObjectName" : { - "type" : "keyword" - }, - "ObjectProperties" : { - "type" : "keyword" - }, - "ObjectServer" : { - "type" : "keyword" - }, - "ObjectType" : { - "type" : "keyword" - }, - "ObjectValueName" : { - "type" : "keyword" - }, - "OemInformation" : { - "type" : "keyword" - }, - "OldMaxUsers" : { - "type" : "keyword" - }, - "OldRemark" : { - "type" : "keyword" - }, - "OldSD" : { - "type" : "keyword" - }, - "OldSd" : { - "type" : "keyword" - }, - "OldShareFlags" : { - "type" : "keyword" - }, - "OldTargetUserName" : { - "type" : "keyword" - }, - "OldUacValue" : { - "type" : "keyword" - }, - "OldValue" : { - "type" : "keyword" - }, - "OldValueType" : { - "type" : "keyword" - }, - "OpCorrelationID" : { - "type" : "keyword" - }, - "Operation" : { - "type" : "keyword" - }, - "OperationId" : { - "type" : "keyword" - }, - "OperationMode" : { - "type" : "keyword" - }, - "OperationName" : { - "type" : "keyword" - }, - "OperationType" : { - "type" : "keyword" - }, - "Options" : { - "type" : "keyword" - }, - "OutboundSpi" : { - "type" : "keyword" - }, - "PackageName" : { - "type" : "keyword" - }, - "ParentProcessName" : { - "type" : "keyword" - }, - "PasswordHistoryLength" : { - "type" : "keyword" - }, - "PasswordLastSet" : { - "type" : "keyword" - }, - "PasswordProperties" : { - "type" : "keyword" - }, - "PeerMac" : { - "type" : "keyword" - }, - "PeerPrivateAddress" : { - "type" : "keyword" - }, - "Policy" : { - "type" : "keyword" - }, - "PreAuthType" : { - "type" : "keyword" - }, - "PreviousTime" : { - "type" : "date" - }, - "PrimaryGroupId" : { - "type" : "keyword" - }, - "PrivilegeList" : { - "type" : "keyword" - }, - "ProcessID" : { - "type" : "keyword" - }, - "ProcessId" : { - "type" : "keyword" - }, - "ProcessName" : { - "type" : "keyword" - }, - "ProductName" : { - "type" : "keyword" - }, - "Profile" : { - "type" : "keyword" - }, - "ProfileChanged" : { - "type" : "keyword" - }, - "ProfilePath" : { - "type" : "keyword" - }, - "ProfileUsed" : { - "type" : "keyword" - }, - "Profiles" : { - "type" : "keyword" - }, - "Properties" : { - "type" : "keyword" - }, - "Protocol" : { - "type" : "keyword" - }, - "ProviderContextKey" : { - "type" : "keyword" - }, - "ProviderContextName" : { - "type" : "keyword" - }, - "ProviderContextType" : { - "type" : "keyword" - }, - "ProviderKey" : { - "type" : "keyword" - }, - "ProviderName" : { - "type" : "keyword" - }, - "ProviderType" : { - "type" : "keyword" - }, - "ProxyPolicyName" : { - "type" : "keyword" - }, - "PuaCount" : { - "type" : "keyword" - }, - "PuaPolicyId" : { - "type" : "keyword" - }, - "PublisherGuid" : { - "type" : "keyword" - }, - "PublisherID" : { - "type" : "keyword" - }, - "PublisherName" : { - "type" : "keyword" - }, - "QMFilterID" : { - "type" : "keyword" - }, - "QMLimit" : { - "type" : "keyword" - }, - "QuarantineHelpURL" : { - "type" : "keyword" - }, - "QuarantineSessionID" : { - "type" : "keyword" - }, - "QuarantineSessionIdentifier" : { - "type" : "keyword" - }, - "QuarantineState" : { - "type" : "keyword" - }, - "QuarantineSystemHealthResult" : { - "type" : "keyword" - }, - "QuickModeSaId" : { - "type" : "keyword" - }, - "Reason" : { - "type" : "keyword" - }, - "ReasonCode" : { - "type" : "keyword" - }, - "ReasonForRejection" : { - "type" : "keyword" - }, - "ReasonText" : { - "type" : "keyword" - }, - "RecoveryKeyId" : { - "type" : "keyword" - }, - "RecoveryReason" : { - "type" : "keyword" - }, - "RecoveryServer" : { - "type" : "keyword" - }, - "RelativeTargetName" : { - "type" : "keyword" - }, - "RemoteAddress" : { - "type" : "keyword" - }, - "RemoteAddressMask" : { - "type" : "keyword" - }, - "RemoteAdminEnabled" : { - "type" : "keyword" - }, - "RemoteEventLogging" : { - "type" : "keyword" - }, - "RemoteKeyModPort" : { - "type" : "keyword" - }, - "RemoteMMPrincipalName" : { - "type" : "keyword" - }, - "RemoteMachineID" : { - "type" : "keyword" - }, - "RemotePort" : { - "type" : "keyword" - }, - "RemotePrivateAddress" : { - "type" : "keyword" - }, - "RemoteTunnelEndpoint" : { - "type" : "keyword" - }, - "RemoteUserID" : { - "type" : "keyword" - }, - "RequestId" : { - "type" : "keyword" - }, - "RequestType" : { - "type" : "keyword" - }, - "Requester" : { - "type" : "keyword" - }, - "ResourceAttributes" : { - "type" : "keyword" - }, - "ResourceManager" : { - "type" : "keyword" - }, - "ResponderCookie" : { - "type" : "keyword" - }, - "RestrictedAdminMode" : { - "type" : "keyword" - }, - "RestrictedSidCount" : { - "type" : "keyword" - }, - "ReturnCode" : { - "type" : "keyword" - }, - "Role" : { - "type" : "keyword" - }, - "RuleAttr" : { - "type" : "keyword" - }, - "RuleId" : { - "type" : "keyword" - }, - "RuleName" : { - "type" : "keyword" - }, - "SPI" : { - "type" : "keyword" - }, - "SSID" : { - "type" : "keyword" - }, - "SamAccountName" : { - "type" : "keyword" - }, - "ScopeName" : { - "type" : "keyword" - }, - "ScriptPath" : { - "type" : "keyword" - }, - "SecurityDescriptor" : { - "type" : "keyword" - }, - "SecurityPackageName" : { - "type" : "keyword" - }, - "Service" : { - "type" : "keyword" - }, - "ServiceAccount" : { - "type" : "keyword" - }, - "ServiceFileName" : { - "type" : "keyword" - }, - "ServiceName" : { - "type" : "keyword" - }, - "ServicePrincipalNames" : { - "type" : "keyword" - }, - "ServiceSid" : { - "type" : "keyword" - }, - "ServiceStartType" : { - "type" : "keyword" - }, - "ServiceType" : { - "type" : "keyword" - }, - "SessionID" : { - "type" : "keyword" - }, - "SessionId" : { - "type" : "keyword" - }, - "SessionName" : { - "type" : "keyword" - }, - "SettingType" : { - "type" : "keyword" - }, - "SettingValue" : { - "type" : "keyword" - }, - "ShareLocalPath" : { - "type" : "keyword" - }, - "ShareName" : { - "type" : "keyword" - }, - "SidHistory" : { - "type" : "keyword" - }, - "SourceAddr" : { - "type" : "keyword" - }, - "SourceAddress" : { - "type" : "keyword" - }, - "SourceDRA" : { - "type" : "keyword" - }, - "SourceHandleId" : { - "type" : "keyword" - }, - "SourcePort" : { - "type" : "keyword" - }, - "SourceProcessId" : { - "type" : "keyword" - }, - "StartUSN" : { - "type" : "keyword" - }, - "State" : { - "type" : "keyword" - }, - "Status" : { - "type" : "keyword" - }, - "StatusCode" : { - "type" : "keyword" - }, - "StoreUrl" : { - "type" : "keyword" - }, - "SubLayerKey" : { - "type" : "keyword" - }, - "SubLayerName" : { - "type" : "keyword" - }, - "SubLayerType" : { - "type" : "keyword" - }, - "SubStatus" : { - "type" : "keyword" - }, - "SubcategoryGuid" : { - "type" : "keyword" - }, - "SubcategoryId" : { - "type" : "keyword" - }, - "Subject" : { - "type" : "keyword" - }, - "SubjectDomainName" : { - "type" : "keyword" - }, - "SubjectKeyIdentifier" : { - "type" : "keyword" - }, - "SubjectLogonId" : { - "type" : "keyword" - }, - "SubjectMachineName" : { - "type" : "keyword" - }, - "SubjectMachineSID" : { - "type" : "keyword" - }, - "SubjectUserDomainName" : { - "type" : "keyword" - }, - "SubjectUserName" : { - "type" : "keyword" - }, - "SubjectUserSid" : { - "type" : "keyword" - }, - "TargetDomainName" : { - "type" : "keyword" - }, - "TargetHandleId" : { - "type" : "keyword" - }, - "TargetInfo" : { - "type" : "keyword" - }, - "TargetLinkedLogonId" : { - "type" : "keyword" - }, - "TargetLogonGuid" : { - "type" : "keyword" - }, - "TargetLogonId" : { - "type" : "keyword" - }, - "TargetOutboundDomainName" : { - "type" : "keyword" - }, - "TargetOutboundUserName" : { - "type" : "keyword" - }, - "TargetProcessId" : { - "type" : "keyword" - }, - "TargetServerName" : { - "type" : "keyword" - }, - "TargetSid" : { - "type" : "keyword" - }, - "TargetUserName" : { - "type" : "keyword" - }, - "TargetUserSid" : { - "type" : "keyword" - }, - "TaskContent" : { - "type" : "keyword" - }, - "TaskContentNew" : { - "type" : "keyword" - }, - "TaskName" : { - "type" : "keyword" - }, - "TemplateContent" : { - "type" : "keyword" - }, - "TemplateDSObjectFQDN" : { - "type" : "keyword" - }, - "TemplateInternalName" : { - "type" : "keyword" - }, - "TemplateOID" : { - "type" : "keyword" - }, - "TemplateSchemaVersion" : { - "type" : "keyword" - }, - "TemplateVersion" : { - "type" : "keyword" - }, - "TestSigning" : { - "type" : "keyword" - }, - "TicketEncryptionType" : { - "type" : "keyword" - }, - "TicketOptions" : { - "type" : "keyword" - }, - "TokenElevationType" : { - "type" : "keyword" - }, - "TrafficSelectorId" : { - "type" : "keyword" - }, - "TransactionId" : { - "type" : "keyword" - }, - "TransmittedServices" : { - "type" : "keyword" - }, - "TransportFilterId" : { - "type" : "keyword" - }, - "TreeDelete" : { - "type" : "keyword" - }, - "TunnelId" : { - "type" : "keyword" - }, - "UserAccountControl" : { - "type" : "keyword" - }, - "UserName" : { - "type" : "keyword" - }, - "UserParameters" : { - "type" : "keyword" - }, - "UserPrincipalName" : { - "type" : "keyword" - }, - "UserSid" : { - "type" : "keyword" - }, - "UserWorkstations" : { - "type" : "keyword" - }, - "VirtualAccount" : { - "type" : "keyword" - }, - "VsmLaunchType" : { - "type" : "keyword" - }, - "Weight" : { - "type" : "keyword" - }, - "Workstation" : { - "type" : "keyword" - }, - "WorkstationName" : { - "type" : "keyword" - }, - "param1" : { - "type" : "keyword" - }, - "param2" : { - "type" : "keyword" - }, - "param3" : { - "type" : "keyword" - }, - "param4" : { - "type" : "keyword" - }, - "param5" : { - "type" : "keyword" - }, - "param6" : { - "type" : "keyword" - }, - "param7" : { - "type" : "keyword" - }, - "param8" : { - "type" : "keyword" - }, - "param9" : { - "type" : "keyword" - }, - "xml_name" : { - "type" : "keyword" - } - } - }, - "eventlog" : { - "properties" : { - "computer" : { - "type" : "keyword" - }, - "eventID" : { - "type" : "long" - }, - "event_recordID" : { - "type" : "long" - }, - "keywords" : { - "type" : "keyword" - }, - "log_name" : { - "type" : "keyword" - }, - "processID" : { - "type" : "long" - }, - "source" : { - "type" : "keyword" - }, - "task_category" : { - "type" : "keyword" - }, - "threadID" : { - "type" : "long" - } - } - }, - "hostIP" : { - "type" : "ip" - }, - "hostIP_geo" : { - "dynamic" : "true", - "properties" : { - "location" : { - "type" : "geo_point" - }, - "postal_code" : { - "type" : "keyword" - } - } - }, - "hostname" : { - "type" : "keyword" - }, - "message" : { - "type" : "text", - "norms" : false, - "fields" : { - "raw" : { - "type" : "keyword", - "ignore_above" : 512 - } - } - }, - "message_error" : { - "type" : "text", - "norms" : false, - "fields" : { - "raw" : { - "type" : "keyword", - "ignore_above" : 512 - } - } - }, - "meta" : { - "include_in_all" : false, - "properties" : { - "beat_type" : { - "type" : "keyword" - }, - "beat_version" : { - "type" : "keyword" - }, - "diff_indexer_shipper_ms" : { - "type" : "long" - }, - "diff_shipper_timestamp_ms" : { - "type" : "long" - }, - "kafka_topic" : { - "type" : "keyword" - }, - "processed_at_indexer" : { - "type" : "keyword" - }, - "processed_at_shipper" : { - "type" : "keyword" - }, - "received_at_indexer" : { - "type" : "date" - }, - "received_at_shipper" : { - "type" : "date" - }, - "received_from" : { - "type" : "keyword" - } - } - }, - "net" : { - "properties" : { - "dst_ip" : { - "type" : "ip" - }, - "dst_ip_geo" : { - "properties" : { - "location" : { - "type" : "geo_point" - } - } - }, - "dst_nat_ip" : { - "type" : "ip" - }, - "dst_nat_port" : { - "type" : "long" - }, - "dst_port" : { - "type" : "long" - }, - "src_domain" : { - "type" : "keyword" - }, - "src_ip" : { - "type" : "ip" - }, - "src_ip_geo" : { - "properties" : { - "location" : { - "type" : "geo_point" - } - } - }, - "src_nat_ip" : { - "type" : "ip" - }, - "src_nat_port" : { - "type" : "long" - }, - "src_port" : { - "type" : "long" - }, - "src_user" : { - "type" : "keyword" - }, - "src_user_department" : { - "type" : "keyword" - }, - "src_user_department_number" : { - "type" : "keyword" - }, - "src_user_display_name" : { - "type" : "keyword" - }, - "src_user_title" : { - "type" : "keyword" - } - } - }, - "severity" : { - "type" : "keyword" - }, - "tags" : { - "type" : "text", - "norms" : false, - "fields" : { - "raw" : { - "type" : "keyword", - "ignore_above" : 512 - } - } - }, - "type" : { - "type" : "keyword" - }, - "user" : { - "type" : "object" - } -} From 6b8177c5719eb33dbaba1a7cebbb537b67c2f75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 25 Nov 2024 17:20:42 +0100 Subject: [PATCH 06/24] iter --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index c0977cd44ac7f..dd08107bd67fb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -10,6 +10,7 @@ package org.elasticsearch.test.rest; import io.netty.handler.codec.http.HttpMethod; + import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -87,7 +88,6 @@ import org.junit.AfterClass; import org.junit.Before; -import javax.net.ssl.SSLContext; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -126,6 +126,8 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import javax.net.ssl.SSLContext; + import static java.util.Collections.sort; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; From 33f3f36dd968e4b65234a685487c7207aefe932e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 25 Nov 2024 22:45:06 +0100 Subject: [PATCH 07/24] Use system property for version --- x-pack/qa/repository-old-versions-7x/build.gradle | 4 +++- .../java/org/elasticsearch/oldrepos7x/OldMappingsIT.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/qa/repository-old-versions-7x/build.gradle b/x-pack/qa/repository-old-versions-7x/build.gradle index 2dcbd610cfe78..f9530ff1f46c3 100644 --- a/x-pack/qa/repository-old-versions-7x/build.gradle +++ b/x-pack/qa/repository-old-versions-7x/build.gradle @@ -4,6 +4,8 @@ import org.elasticsearch.gradle.Version apply plugin: 'elasticsearch.internal-java-rest-test' tasks.named("javaRestTest").configure { + def versionString = "7.17.25" + systemProperty 'tests.old_cluster_version', versionString usesDefaultDistribution() - usesBwcDistribution(Version.fromString("7.17.25")) + usesBwcDistribution(Version.fromString(versionString)) } \ No newline at end of file diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java index ca1c296bd59be..689373d3bc02c 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java @@ -57,7 +57,7 @@ public class OldMappingsIT extends ESRestTestCase { .build(); public static ElasticsearchCluster oldCluster = ElasticsearchCluster.local() - .version(Version.fromString("7.17.25")) + .version(Version.fromString(System.getProperty("tests.old_cluster_version"))) .distribution(DistributionType.DEFAULT) .setting("xpack.security.enabled", "false") .setting("xpack.license.self_generated.type", "trial") From dce64b24ba8d57ac83799d66d8d53cdf8f532d9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 25 Nov 2024 22:58:02 +0100 Subject: [PATCH 08/24] Add BWCLucene86Codec --- .../xpack/lucene/bwc/codecs/BWCCodec.java | 9 +- .../bwc/codecs/lucene86/BWCLucene86Codec.java | 153 ++++++++++++++++++ 2 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 1f367147a582a..432974dc256eb 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; +import org.apache.lucene.backward_codecs.lucene86.Lucene86Codec; import org.apache.lucene.backward_codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldInfosFormat; @@ -27,6 +28,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec; import java.io.IOException; @@ -122,7 +124,12 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) { public static SegmentInfo wrap(SegmentInfo segmentInfo) { // special handling for Lucene87Codec (which is currently bundled with Lucene) // Use BWCLucene87Codec instead as that one extends BWCCodec (similar to all other older codecs) - final Codec codec = segmentInfo.getCodec() instanceof Lucene87Codec ? new BWCLucene87Codec() : segmentInfo.getCodec(); + Codec codec = segmentInfo.getCodec(); + if (codec instanceof Lucene86Codec) { + codec = new BWCLucene86Codec(); + } else if (codec instanceof Lucene87Codec) { + codec = new BWCLucene87Codec(); + } final SegmentInfo segmentInfo1 = new SegmentInfo( segmentInfo.dir, // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java new file mode 100644 index 0000000000000..1dc06706eaa93 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat; +import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + +import java.util.Objects; + +public class BWCLucene86Codec extends BWCCodec { + + private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86PointsFormat(); + private final PostingsFormat defaultFormat; + + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return BWCLucene86Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return BWCLucene86Codec.this.getDocValuesFormatForField(field); + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + + /** Instantiates a new codec. */ + public BWCLucene86Codec() { + super("BWCLucene87Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(Lucene50StoredFieldsFormat.Mode.BEST_SPEED)); + this.defaultFormat = new Lucene84PostingsFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final KnnVectorsFormat knnVectorsFormat() { + return KnnVectorsFormat.EMPTY; + } + + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene84". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene80". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); + + private final NormsFormat normsFormat = new Lucene80NormsFormat(); + + @Override + public NormsFormat normsFormat() { + return normsFormat; + } + +} From 2f3b6d08ea5f322bbbb95a1871f7b3a766daa3ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 25 Nov 2024 23:29:01 +0100 Subject: [PATCH 09/24] Fix codec name --- .../xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java index 1dc06706eaa93..4372e44f8171c 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -61,7 +61,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) { /** Instantiates a new codec. */ public BWCLucene86Codec() { - super("BWCLucene87Codec"); + super("BWCLucene86Codec"); this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(Lucene50StoredFieldsFormat.Mode.BEST_SPEED)); this.defaultFormat = new Lucene84PostingsFormat(); } From feaee2824f920a0ba806ba160abbd3ac214ebf5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 26 Nov 2024 09:07:59 +0100 Subject: [PATCH 10/24] Change version to 7.9.0 --- x-pack/qa/repository-old-versions-7x/build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/qa/repository-old-versions-7x/build.gradle b/x-pack/qa/repository-old-versions-7x/build.gradle index f9530ff1f46c3..1577742785b1d 100644 --- a/x-pack/qa/repository-old-versions-7x/build.gradle +++ b/x-pack/qa/repository-old-versions-7x/build.gradle @@ -4,8 +4,9 @@ import org.elasticsearch.gradle.Version apply plugin: 'elasticsearch.internal-java-rest-test' tasks.named("javaRestTest").configure { - def versionString = "7.17.25" + // def versionString = "7.17.25" + def versionString = "7.9.0" systemProperty 'tests.old_cluster_version', versionString usesDefaultDistribution() usesBwcDistribution(Version.fromString(versionString)) -} \ No newline at end of file +} From 41d69c7d7d0efd1008a1692e13d044267ecc2af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 26 Nov 2024 12:17:35 +0100 Subject: [PATCH 11/24] Add modified version of OldRepositoryAccessIT --- .../repository-old-versions-7x/build.gradle | 4 +- .../oldrepos7x/OldRepositoryAccessIT.java | 535 ++++++++++++++++++ 2 files changed, 537 insertions(+), 2 deletions(-) create mode 100644 x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java diff --git a/x-pack/qa/repository-old-versions-7x/build.gradle b/x-pack/qa/repository-old-versions-7x/build.gradle index 1577742785b1d..7d0e047dc8603 100644 --- a/x-pack/qa/repository-old-versions-7x/build.gradle +++ b/x-pack/qa/repository-old-versions-7x/build.gradle @@ -4,8 +4,8 @@ import org.elasticsearch.gradle.Version apply plugin: 'elasticsearch.internal-java-rest-test' tasks.named("javaRestTest").configure { - // def versionString = "7.17.25" - def versionString = "7.9.0" + def versionString = "7.17.25" + // def versionString = "7.9.0" systemProperty 'tests.old_cluster_version', versionString usesDefaultDistribution() usesBwcDistribution(Version.fromString(versionString)) diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java new file mode 100644 index 0000000000000..afe2f2b3cfa10 --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -0,0 +1,535 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos7x; + +import org.apache.http.HttpHost; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchResponseUtils; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; + +public class OldRepositoryAccessIT extends ESRestTestCase { + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + public static TemporaryFolder repoDirectory = new TemporaryFolder(); + + public static ElasticsearchCluster currentCluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .nodes(2) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.ml.enabled", "false") + .setting("path.repo", () -> repoDirectory.getRoot().getPath()) + .setting("xpack.searchable.snapshot.shared_cache.size", "16MB") + .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") + .build(); + + public static ElasticsearchCluster oldCluster = ElasticsearchCluster.local() + .version(org.elasticsearch.test.cluster.util.Version.fromString(System.getProperty("tests.old_cluster_version"))) + .distribution(DistributionType.DEFAULT) + .nodes(2) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.ml.enabled", "false") + .setting("path.repo", () -> repoDirectory.getRoot().getPath()) + .setting("xpack.searchable.snapshot.shared_cache.size", "16MB") + .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(oldCluster).around(currentCluster); + + private static boolean repoRestored = false; + private static String repoLocation; + private static String repoName; + private static String snapshotName; + + @Override + protected String getTestRestCluster() { + return currentCluster.getHttpAddresses(); + } + + public void testOldRepoAccess() throws IOException { + runTest(false); + } + + // public void testOldSourceOnlyRepoAccess() throws IOException { + // runTest(true); + // } + + public void runTest(boolean sourceOnlyRepository) throws IOException { + // boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); + String repoLocation = repoDirectory.getRoot().getPath(); + repoLocation = PathUtils.get(repoLocation).resolve("source_only_" + sourceOnlyRepository).toString(); + Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); + assumeTrue( + "source only repositories only supported since ES 6.5.0", + sourceOnlyRepository == false || oldVersion.onOrAfter(Version.fromString("6.5.0")) + ); + + assertThat("Index version should be added to archive tests", oldVersion, lessThan(Version.V_8_10_0)); + IndexVersion indexVersion = IndexVersion.fromId(oldVersion.id); + + System.out.println("Here"); + String httpAddresses = oldCluster.getHttpAddresses(); + System.out.println(httpAddresses); + List oldClusterHosts = parseClusterHosts(httpAddresses); + System.out.println("oldCH: " + oldClusterHosts); + String indexName; + if (sourceOnlyRepository) { + indexName = "source_only_test_index"; + } else { + indexName = "test_index"; + } + int numDocs = 10; + int extraDocs = 1; + try (RestClient oldEs = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build()) { + // if (afterRestart == false) { + beforeRestart(sourceOnlyRepository, repoLocation, oldVersion, indexVersion, numDocs, extraDocs, oldEs, indexName); + // } else { + // afterRestart(indexName); + // } + } + } + + // private void afterRestart(String indexName) throws IOException { + // ensureGreen("restored_" + indexName); + // ensureGreen("mounted_full_copy_" + indexName); + // ensureGreen("mounted_shared_cache_" + indexName); + // } + + private void beforeRestart( + boolean sourceOnlyRepository, + String repoLocation, + Version oldVersion, + IndexVersion indexVersion, + int numDocs, + int extraDocs, + RestClient oldEs, + String indexName + ) throws IOException { + String repoName = "repo_" + indexName; + String snapshotName = "snap_" + indexName; + Request createIndex = new Request("PUT", "/" + indexName); + int numberOfShards = randomIntBetween(1, 3); + + XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject().startObject("settings"); + settingsBuilder.field("index.number_of_shards", numberOfShards); + + // 6.5.0 started using soft-deletes, but it was only enabled by default on 7.0 + if (oldVersion.onOrAfter(Version.fromString("6.5.0")) && oldVersion.before(Version.fromString("7.0.0")) && randomBoolean()) { + settingsBuilder.field("index.soft_deletes.enabled", true); + } + + settingsBuilder.endObject().endObject(); + + createIndex.setJsonEntity(Strings.toString(settingsBuilder)); + assertOK(oldEs.performRequest(createIndex)); + + // TODO maybe go bak to using multiple types for ES versions < 6.0.0 if we migrate them to this qa project + String type = "_doc"; // + Set expectedIds = new HashSet<>(); + for (int i = 0; i < numDocs + extraDocs; i++) { + String id = "testdoc" + i; + expectedIds.add(id); + Request doc = new Request("PUT", "/" + indexName + "/" + type + "/" + id); + doc.addParameter("refresh", "true"); + doc.setJsonEntity(sourceForDoc(i)); + assertOK(oldEs.performRequest(doc)); + } + + for (int i = 0; i < extraDocs; i++) { + String id = randomFrom(expectedIds); + expectedIds.remove(id); + Request doc = new Request("DELETE", "/" + indexName + "/" + type + "/" + id); + doc.addParameter("refresh", "true"); + oldEs.performRequest(doc); + } + + // register repo on old ES and take snapshot + Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest.setJsonEntity(sourceOnlyRepository ? Strings.format(""" + {"type":"source","settings":{"location":"%s","delegate_type":"fs"}} + """, repoLocation) : Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(oldEs.performRequest(createRepoRequest)); + + Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); + assertOK(oldEs.performRequest(createSnapshotRequest)); + + // register repo on new ES + Settings.Builder repoSettingsBuilder = Settings.builder().put("location", repoLocation); + if (sourceOnlyRepository) { + repoSettingsBuilder.put("delegate_type", "fs"); + } + Request createRepo = new Request("PUT", "/_snapshot/" + repoName); + createRepo.setJsonEntity( + Strings.toString( + new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).type(sourceOnlyRepository ? "source" : "fs") + .settings(repoSettingsBuilder.build()) + ) + ); + assertAcknowledged(client().performRequest(createRepo)); + + // list snapshots on new ES + Request getSnaps = new Request("GET", "/_snapshot/" + repoName + "/_all"); + Response getResponse = client().performRequest(getSnaps); + ObjectPath getResp = ObjectPath.createFromResponse(getResponse); + assertThat(getResp.evaluate("total"), equalTo(1)); + assertThat(getResp.evaluate("snapshots.0.snapshot"), equalTo(snapshotName)); + assertThat(getResp.evaluate("snapshots.0.repository"), equalTo(repoName)); + assertThat(getResp.evaluate("snapshots.0.indices"), contains(indexName)); + assertThat(getResp.evaluate("snapshots.0.state"), equalTo(SnapshotState.SUCCESS.toString())); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.successful")); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.total")); + assertEquals(0, (int) getResp.evaluate("snapshots.0.shards.failed")); + assertEquals(indexVersion.toReleaseVersion(), getResp.evaluate("snapshots.0.version")); + + // list specific snapshot on new ES + getSnaps = new Request("GET", "/_snapshot/" + repoName + "/" + snapshotName); + getResponse = client().performRequest(getSnaps); + getResp = ObjectPath.createFromResponse(getResponse); + assertThat(getResp.evaluate("total"), equalTo(1)); + assertThat(getResp.evaluate("snapshots.0.snapshot"), equalTo(snapshotName)); + assertThat(getResp.evaluate("snapshots.0.repository"), equalTo(repoName)); + assertThat(getResp.evaluate("snapshots.0.indices"), contains(indexName)); + assertThat(getResp.evaluate("snapshots.0.state"), equalTo(SnapshotState.SUCCESS.toString())); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.successful")); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.total")); + assertEquals(0, (int) getResp.evaluate("snapshots.0.shards.failed")); + assertEquals(indexVersion.toReleaseVersion(), getResp.evaluate("snapshots.0.version")); + + // list advanced snapshot info on new ES + getSnaps = new Request("GET", "/_snapshot/" + repoName + "/" + snapshotName + "/_status"); + getResponse = client().performRequest(getSnaps); + getResp = ObjectPath.createFromResponse(getResponse); + assertThat(((List) getResp.evaluate("snapshots")).size(), equalTo(1)); + assertThat(getResp.evaluate("snapshots.0.snapshot"), equalTo(snapshotName)); + assertThat(getResp.evaluate("snapshots.0.repository"), equalTo(repoName)); + assertThat(((Map) getResp.evaluate("snapshots.0.indices")).keySet(), contains(indexName)); + assertThat(getResp.evaluate("snapshots.0.state"), equalTo(SnapshotState.SUCCESS.toString())); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards_stats.done")); + assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards_stats.total")); + assertEquals(0, (int) getResp.evaluate("snapshots.0.shards_stats.failed")); + assertThat(getResp.evaluate("snapshots.0.stats.total.size_in_bytes"), greaterThan(0)); + assertThat(getResp.evaluate("snapshots.0.stats.total.file_count"), greaterThan(0)); + + // restore / mount and check whether searches work + restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); + + // close indices + closeIndex(client(), "restored_" + indexName); + closeIndex(client(), "mounted_full_copy_" + indexName); + closeIndex(client(), "mounted_shared_cache_" + indexName); + + // restore / mount again + restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); + } + + private static String sourceForDoc(int i) { + return "{\"test\":\"test" + i + "\",\"val\":" + i + ",\"create_date\":\"2020-01-" + Strings.format("%02d", i + 1) + "\"}"; + } + + private void restoreMountAndVerify( + int numDocs, + Set expectedIds, + int numberOfShards, + boolean sourceOnlyRepository, + Version oldVersion, + String indexName, + String repoName, + String snapshotName + ) throws IOException { + // restore index + Request restoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); + restoreRequest.setJsonEntity( + Strings.toString( + new RestoreSnapshotRequest(TEST_REQUEST_TIMEOUT).indices(indexName).renamePattern("(.+)").renameReplacement("restored_$1") + ) + ); + restoreRequest.addParameter("wait_for_completion", "true"); + Response restoreResponse = client().performRequest(restoreRequest); + ObjectPath restore = ObjectPath.createFromResponse(restoreResponse); + assertEquals(numberOfShards, (int) restore.evaluate("snapshot.shards.total")); + assertEquals(numberOfShards, (int) restore.evaluate("snapshot.shards.successful")); + + ensureGreen("restored_" + indexName); + + String restoredIndex = "restored_" + indexName; + var response = responseAsMap(client().performRequest(new Request("GET", "/" + restoredIndex + "/_mapping"))); + Map mapping = ObjectPath.evaluate(response, restoredIndex + ".mappings"); + logger.info("mapping for {}: {}", restoredIndex, mapping); + assertThat(mapping, hasKey("_meta")); + assertThat(mapping.get("_meta"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map meta = (Map) mapping.get("_meta"); + assertThat(meta, hasKey("legacy_mappings")); + assertThat(meta.get("legacy_mappings"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map legacyMappings = (Map) meta.get("legacy_mappings"); + assertThat(legacyMappings.keySet(), not(empty())); + for (Map.Entry entry : legacyMappings.entrySet()) { + String type = entry.getKey(); + assertThat(type, startsWith("_doc")); + assertThat(entry.getValue(), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map legacyMapping = (Map) entry.getValue(); + assertThat(legacyMapping, hasKey("properties")); + assertThat(legacyMapping.get("properties"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map propertiesMapping = (Map) legacyMapping.get("properties"); + assertThat(propertiesMapping, hasKey("val")); + assertThat(propertiesMapping.get("val"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map valMapping = (Map) propertiesMapping.get("val"); + assertThat(valMapping, hasKey("type")); + assertEquals("long", valMapping.get("type")); + } + + // run a search against the index + assertDocs("restored_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); + + // mount as full copy searchable snapshot + Request mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); + mountRequest.setJsonEntity( + "{\"index\": \"" + + indexName + + "\",\"renamed_index\": \"mounted_full_copy_" + + indexName + + "\",\"index_settings\": {\"index.number_of_replicas\": 1}}" + ); + mountRequest.addParameter("wait_for_completion", "true"); + ObjectPath mountResponse = ObjectPath.createFromResponse(client().performRequest(mountRequest)); + assertNotNull(mountResponse.evaluate("snapshot")); + assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.total")); + assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.successful")); + + ensureGreen("mounted_full_copy_" + indexName); + + // run a search against the index + assertDocs("mounted_full_copy_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); + + // mount as shared cache searchable snapshot + mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); + mountRequest.setJsonEntity("{\"index\": \"" + indexName + "\",\"renamed_index\": \"mounted_shared_cache_" + indexName + "\"}"); + mountRequest.addParameter("wait_for_completion", "true"); + mountRequest.addParameter("storage", "shared_cache"); + mountResponse = ObjectPath.createFromResponse(client().performRequest(mountRequest)); + assertNotNull(mountResponse.evaluate("snapshot")); + System.out.println("---> " + Strings.toString(mountResponse.toXContentBuilder(XContentType.JSON.xContent()))); + assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.total")); + assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.successful")); + + // run a search against the index + assertDocs("mounted_shared_cache_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); + } + + private void assertDocs( + String index, + int numDocs, + Set expectedIds, + boolean sourceOnlyRepository, + Version oldVersion, + int numberOfShards + ) throws IOException { + RequestOptions requestOptions = RequestOptions.DEFAULT; + + // run a search against the index + SearchResponse searchResponse = search(index, null, requestOptions); + try { + logger.info(searchResponse); + // check hit count + assertEquals(numDocs, searchResponse.getHits().getTotalHits().value()); + // check that _index is properly set + assertTrue(Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getIndex).allMatch(index::equals)); + // check that all _ids are there + assertEquals(expectedIds, Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toSet())); + // check that _source is present + assertTrue(Arrays.stream(searchResponse.getHits().getHits()).allMatch(SearchHit::hasSource)); + // check that correct _source present for each document + for (SearchHit h : searchResponse.getHits().getHits()) { + assertEquals(sourceForDoc(getIdAsNumeric(h.getId())), h.getSourceAsString()); + } + } finally { + searchResponse.decRef(); + } + + String id = randomFrom(expectedIds); + int num = getIdAsNumeric(id); + // run a search using runtime fields against the index + searchResponse = search( + index, + SearchSourceBuilder.searchSource() + .query(QueryBuilders.matchQuery("val", num)) + .runtimeMappings(Map.of("val", Map.of("type", "long"))), + requestOptions + ); + try { + logger.info(searchResponse); + assertEquals(1, searchResponse.getHits().getTotalHits().value()); + assertEquals(id, searchResponse.getHits().getHits()[0].getId()); + assertEquals(sourceForDoc(num), searchResponse.getHits().getHits()[0].getSourceAsString()); + } finally { + searchResponse.decRef(); + } + + if (sourceOnlyRepository == false) { + // search using reverse sort on val + searchResponse = search( + index, + SearchSourceBuilder.searchSource() + .query(QueryBuilders.matchAllQuery()) + .sort(SortBuilders.fieldSort("val").order(SortOrder.DESC)), + requestOptions + ); + try { + logger.info(searchResponse); + // check sort order + assertEquals( + expectedIds.stream().sorted(Comparator.comparingInt(this::getIdAsNumeric).reversed()).toList(), + Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).toList() + ); + } finally { + searchResponse.decRef(); + } + + // look up postings + searchResponse = search( + index, + SearchSourceBuilder.searchSource().query(QueryBuilders.matchQuery("test", "test" + num)), + requestOptions + ); + try { + logger.info(searchResponse); + // check match + ElasticsearchAssertions.assertSearchHits(searchResponse, id); + } finally { + searchResponse.decRef(); + } + + // if (oldVersion.before(Version.fromString("6.0.0"))) { + // // search on _type and check that results contain _type information + // String randomType = getType(oldVersion, randomFrom(expectedIds)); + // long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count(); + // searchResponse = search( + // index, + // SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType)), + // requestOptions + // ); + // try { + // logger.info(searchResponse); + // assertEquals(typeCount, searchResponse.getHits().getTotalHits().value()); + // for (SearchHit hit : searchResponse.getHits().getHits()) { + // DocumentField typeField = hit.field("_type"); + // assertNotNull(typeField); + // assertThat(typeField.getValue(), instanceOf(String.class)); + // assertEquals(randomType, typeField.getValue()); + // } + // } finally { + // searchResponse.decRef(); + // } + // } + + assertThat( + expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/" + index + "/_doc/" + id))) + .getMessage(), + containsString("get operations not allowed on a legacy index") + ); + + // check that shards are skipped based on non-matching date + searchResponse = search( + index, + SearchSourceBuilder.searchSource().query(QueryBuilders.rangeQuery("create_date").from("2020-02-01")), + requestOptions + ); + try { + logger.info(searchResponse); + assertEquals(0, searchResponse.getHits().getTotalHits().value()); + assertEquals(numberOfShards, searchResponse.getSuccessfulShards()); + assertEquals(numberOfShards, searchResponse.getSkippedShards()); + } finally { + searchResponse.decRef(); + } + } + } + + private static SearchResponse search(String index, @Nullable SearchSourceBuilder builder, RequestOptions options) throws IOException { + Request request = new Request("POST", "/" + index + "/_search"); + if (builder != null) { + request.setJsonEntity(builder.toString()); + } + request.setOptions(options); + return SearchResponseUtils.parseSearchResponse(responseAsParser(client().performRequest(request))); + } + + private int getIdAsNumeric(String id) { + return Integer.parseInt(id.substring("testdoc".length())); + } + + private static void closeIndex(RestClient client, String index) throws IOException { + Request request = new Request("POST", "/" + index + "/_close"); + ObjectPath doc = ObjectPath.createFromResponse(client.performRequest(request)); + assertTrue(doc.evaluate("shards_acknowledged")); + } +} From 8b3acb89c67285518cbf0b5465916f77d028ce00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 26 Nov 2024 15:44:04 +0100 Subject: [PATCH 12/24] Adding back test for source_only repo --- .../oldrepos7x/OldRepositoryAccessIT.java | 235 +++++++++++------- 1 file changed, 140 insertions(+), 95 deletions(-) diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java index afe2f2b3cfa10..d160c750e3544 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -37,20 +37,26 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; +import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; +import static java.util.Collections.unmodifiableList; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -64,11 +70,6 @@ public class OldRepositoryAccessIT extends ESRestTestCase { - @Override - protected boolean preserveClusterUponCompletion() { - return true; - } - public static TemporaryFolder repoDirectory = new TemporaryFolder(); public static ElasticsearchCluster currentCluster = ElasticsearchCluster.local() @@ -97,10 +98,103 @@ protected boolean preserveClusterUponCompletion() { @ClassRule public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(oldCluster).around(currentCluster); - private static boolean repoRestored = false; - private static String repoLocation; - private static String repoName; - private static String snapshotName; + private static final String REPO_LOCATION_BASE = "source_only_"; + + private static final Map> expectedTestIds = new HashMap<>(); + + private static final Map expectedNumberOfShards = new HashMap<>(); + + private static final int numDocs = 10; + private static final int extraDocs = 1; + + @BeforeClass + public static void setupOldRepo() throws IOException { + String repoLocationBase = repoDirectory.getRoot().getPath(); + + List oldClusterHosts = parseOldClusterHosts(oldCluster.getHttpAddresses()); + try (RestClient oldEs = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build()) { + for (boolean sourceOnlyRepository : new boolean[] { true, false }) { + String repoLocation = PathUtils.get(repoLocationBase).resolve(REPO_LOCATION_BASE + sourceOnlyRepository).toString(); + Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); + assumeTrue( + "source only repositories only supported since ES 6.5.0", + sourceOnlyRepository == false || oldVersion.onOrAfter(Version.fromString("6.5.0")) + ); + + assertThat("Index version should be added to archive tests", oldVersion, lessThan(Version.V_8_10_0)); + IndexVersion indexVersion = IndexVersion.fromId(oldVersion.id); + String indexName; + if (sourceOnlyRepository) { + indexName = "source_only_test_index"; + } else { + indexName = "test_index"; + } + + String repoName = "repo_" + indexName; + String snapshotName = "snap_" + indexName; + Request createIndex = new Request("PUT", "/" + indexName); + int numShards = randomIntBetween(1, 3); + expectedNumberOfShards.put(repoLocation, numShards); + + XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject().startObject("settings"); + settingsBuilder.field("index.number_of_shards", numShards); + + // 6.5.0 started using soft-deletes, but it was only enabled by default on 7.0 + if (oldVersion.onOrAfter(Version.fromString("6.5.0")) + && oldVersion.before(Version.fromString("7.0.0")) + && randomBoolean()) { + settingsBuilder.field("index.soft_deletes.enabled", true); + } + + settingsBuilder.endObject().endObject(); + + createIndex.setJsonEntity(Strings.toString(settingsBuilder)); + assertOK(oldEs.performRequest(createIndex)); + + // TODO maybe go back to using multiple types for ES versions < 6.0.0 if we migrate them to this qa project + String type = "_doc"; + Set expectedIds = new HashSet<>(); + for (int i = 0; i < numDocs + extraDocs; i++) { + String id = "testdoc" + i; + expectedIds.add(id); + Request doc = new Request("PUT", "/" + indexName + "/" + type + "/" + id); + doc.addParameter("refresh", "true"); + doc.setJsonEntity(sourceForDoc(i)); + assertOK(oldEs.performRequest(doc)); + } + + for (int i = 0; i < extraDocs; i++) { + String id = randomFrom(expectedIds); + expectedIds.remove(id); + Request doc = new Request("DELETE", "/" + indexName + "/" + type + "/" + id); + doc.addParameter("refresh", "true"); + oldEs.performRequest(doc); + } + expectedTestIds.put(repoLocation, expectedIds); + + // register repo on old ES and take snapshot + Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); + createRepoRequest.setJsonEntity(sourceOnlyRepository ? Strings.format(""" + {"type":"source","settings":{"location":"%s","delegate_type":"fs"}} + """, repoLocation) : Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(oldEs.performRequest(createRepoRequest)); + + Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); + assertOK(oldEs.performRequest(createSnapshotRequest)); + } + } + } + + public static Set listFilesUsingJavaIO(String dir) { + return Stream.of(new File(dir).listFiles()) + .filter(file -> file.isDirectory() == false) + .map(File::getName) + .collect(Collectors.toSet()); + } @Override protected String getTestRestCluster() { @@ -111,14 +205,14 @@ public void testOldRepoAccess() throws IOException { runTest(false); } - // public void testOldSourceOnlyRepoAccess() throws IOException { - // runTest(true); - // } + public void testOldSourceOnlyRepoAccess() throws IOException { + runTest(true); + } public void runTest(boolean sourceOnlyRepository) throws IOException { // boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); String repoLocation = repoDirectory.getRoot().getPath(); - repoLocation = PathUtils.get(repoLocation).resolve("source_only_" + sourceOnlyRepository).toString(); + repoLocation = PathUtils.get(repoLocation).resolve(REPO_LOCATION_BASE + sourceOnlyRepository).toString(); Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); assumeTrue( "source only repositories only supported since ES 6.5.0", @@ -128,95 +222,16 @@ public void runTest(boolean sourceOnlyRepository) throws IOException { assertThat("Index version should be added to archive tests", oldVersion, lessThan(Version.V_8_10_0)); IndexVersion indexVersion = IndexVersion.fromId(oldVersion.id); - System.out.println("Here"); - String httpAddresses = oldCluster.getHttpAddresses(); - System.out.println(httpAddresses); - List oldClusterHosts = parseClusterHosts(httpAddresses); - System.out.println("oldCH: " + oldClusterHosts); String indexName; if (sourceOnlyRepository) { indexName = "source_only_test_index"; } else { indexName = "test_index"; } - int numDocs = 10; - int extraDocs = 1; - try (RestClient oldEs = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build()) { - // if (afterRestart == false) { - beforeRestart(sourceOnlyRepository, repoLocation, oldVersion, indexVersion, numDocs, extraDocs, oldEs, indexName); - // } else { - // afterRestart(indexName); - // } - } - } - - // private void afterRestart(String indexName) throws IOException { - // ensureGreen("restored_" + indexName); - // ensureGreen("mounted_full_copy_" + indexName); - // ensureGreen("mounted_shared_cache_" + indexName); - // } - - private void beforeRestart( - boolean sourceOnlyRepository, - String repoLocation, - Version oldVersion, - IndexVersion indexVersion, - int numDocs, - int extraDocs, - RestClient oldEs, - String indexName - ) throws IOException { String repoName = "repo_" + indexName; String snapshotName = "snap_" + indexName; - Request createIndex = new Request("PUT", "/" + indexName); - int numberOfShards = randomIntBetween(1, 3); - - XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject().startObject("settings"); - settingsBuilder.field("index.number_of_shards", numberOfShards); - - // 6.5.0 started using soft-deletes, but it was only enabled by default on 7.0 - if (oldVersion.onOrAfter(Version.fromString("6.5.0")) && oldVersion.before(Version.fromString("7.0.0")) && randomBoolean()) { - settingsBuilder.field("index.soft_deletes.enabled", true); - } - - settingsBuilder.endObject().endObject(); - - createIndex.setJsonEntity(Strings.toString(settingsBuilder)); - assertOK(oldEs.performRequest(createIndex)); - - // TODO maybe go bak to using multiple types for ES versions < 6.0.0 if we migrate them to this qa project - String type = "_doc"; // - Set expectedIds = new HashSet<>(); - for (int i = 0; i < numDocs + extraDocs; i++) { - String id = "testdoc" + i; - expectedIds.add(id); - Request doc = new Request("PUT", "/" + indexName + "/" + type + "/" + id); - doc.addParameter("refresh", "true"); - doc.setJsonEntity(sourceForDoc(i)); - assertOK(oldEs.performRequest(doc)); - } - - for (int i = 0; i < extraDocs; i++) { - String id = randomFrom(expectedIds); - expectedIds.remove(id); - Request doc = new Request("DELETE", "/" + indexName + "/" + type + "/" + id); - doc.addParameter("refresh", "true"); - oldEs.performRequest(doc); - } - - // register repo on old ES and take snapshot - Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest.setJsonEntity(sourceOnlyRepository ? Strings.format(""" - {"type":"source","settings":{"location":"%s","delegate_type":"fs"}} - """, repoLocation) : Strings.format(""" - {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); - assertOK(oldEs.performRequest(createRepoRequest)); - - Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); - createSnapshotRequest.addParameter("wait_for_completion", "true"); - createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); - assertOK(oldEs.performRequest(createSnapshotRequest)); + Set expectedIds = expectedTestIds.get(repoLocation); + int numberOfShards = expectedNumberOfShards.get(repoLocation); // register repo on new ES Settings.Builder repoSettingsBuilder = Settings.builder().put("location", repoLocation); @@ -284,7 +299,22 @@ private void beforeRestart( closeIndex(client(), "mounted_shared_cache_" + indexName); // restore / mount again - restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); + restoreMountAndVerify( + numDocs, + expectedIds, + numberOfShards, + sourceOnlyRepository, + oldVersion, + indexName, + repoName, + snapshotName + ); + + // TODO restart current cluster + + // ensureGreen("restored_" + indexName); + // ensureGreen("mounted_full_copy_" + indexName); + // ensureGreen("mounted_shared_cache_" + indexName); } private static String sourceForDoc(int i) { @@ -532,4 +562,19 @@ private static void closeIndex(RestClient client, String index) throws IOExcepti ObjectPath doc = ObjectPath.createFromResponse(client.performRequest(request)); assertTrue(doc.evaluate("shards_acknowledged")); } + + private static List parseOldClusterHosts(String hostsString) { + String[] stringUrls = hostsString.split(","); + List hosts = new ArrayList<>(stringUrls.length); + for (String stringUrl : stringUrls) { + int portSeparator = stringUrl.lastIndexOf(':'); + if (portSeparator < 0) { + throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]"); + } + String host = stringUrl.substring(0, portSeparator); + int port = Integer.valueOf(stringUrl.substring(portSeparator + 1)); + hosts.add(new HttpHost(host, port)); + } + return unmodifiableList(hosts); + } } From 55ed6d075e1566628f7980b308ec13430a065d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 26 Nov 2024 21:54:43 +0100 Subject: [PATCH 13/24] Add looping over versions --- .../repository-old-versions-7x/build.gradle | 25 ++++++++++++++----- .../oldrepos7x/OldRepositoryAccessIT.java | 11 +------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/x-pack/qa/repository-old-versions-7x/build.gradle b/x-pack/qa/repository-old-versions-7x/build.gradle index 7d0e047dc8603..322e772ac8585 100644 --- a/x-pack/qa/repository-old-versions-7x/build.gradle +++ b/x-pack/qa/repository-old-versions-7x/build.gradle @@ -1,12 +1,25 @@ import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask import org.elasticsearch.gradle.Version apply plugin: 'elasticsearch.internal-java-rest-test' -tasks.named("javaRestTest").configure { - def versionString = "7.17.25" - // def versionString = "7.9.0" - systemProperty 'tests.old_cluster_version', versionString - usesDefaultDistribution() - usesBwcDistribution(Version.fromString(versionString)) +tasks.named("javaRestTest") { + enabled = false +} + +for (String versionString : ['7.9.0', '7.17.25']) { + String versionNoDots = versionString.replace('.', '_') + + tasks.register("javaRestTest#${versionNoDots}", StandaloneRestIntegTestTask) { + systemProperty 'tests.old_cluster_version', versionString + usesDefaultDistribution() + testClassesDirs = sourceSets.javaRestTest.output.classesDirs + classpath = sourceSets.javaRestTest.runtimeClasspath + usesBwcDistribution(Version.fromString(versionString)) + } + + tasks.named("check").configure { + dependsOn "javaRestTest#${versionNoDots}" + } } diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java index d160c750e3544..eaaab928b5a12 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -299,16 +299,7 @@ public void runTest(boolean sourceOnlyRepository) throws IOException { closeIndex(client(), "mounted_shared_cache_" + indexName); // restore / mount again - restoreMountAndVerify( - numDocs, - expectedIds, - numberOfShards, - sourceOnlyRepository, - oldVersion, - indexName, - repoName, - snapshotName - ); + restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); // TODO restart current cluster From 1351e2ed44781ba4190debe94e9393af27b0b74a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 27 Nov 2024 11:41:46 +0100 Subject: [PATCH 14/24] Change looping over version --- .../repository-old-versions-7x/build.gradle | 12 +++- .../oldrepos7x/OldRepositoryAccessIT.java | 64 +++++-------------- 2 files changed, 28 insertions(+), 48 deletions(-) diff --git a/x-pack/qa/repository-old-versions-7x/build.gradle b/x-pack/qa/repository-old-versions-7x/build.gradle index 322e772ac8585..004194c7782f4 100644 --- a/x-pack/qa/repository-old-versions-7x/build.gradle +++ b/x-pack/qa/repository-old-versions-7x/build.gradle @@ -3,12 +3,22 @@ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask import org.elasticsearch.gradle.Version apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.rest-resources' + +restResources { + restApi { + include '_common', 'search' + } + restTests { + includeCore 'search/390_doc_values_search.yml' + } +} tasks.named("javaRestTest") { enabled = false } -for (String versionString : ['7.9.0', '7.17.25']) { +['7.16.3', '7.17.25'].each { versionString -> String versionNoDots = versionString.replace('.', '_') tasks.register("javaRestTest#${versionNoDots}", StandaloneRestIntegTestTask) { diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java index eaaab928b5a12..c5b08319e0949 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -43,7 +43,6 @@ import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -54,7 +53,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.util.Collections.unmodifiableList; import static org.hamcrest.Matchers.contains; @@ -110,12 +108,12 @@ public class OldRepositoryAccessIT extends ESRestTestCase { @BeforeClass public static void setupOldRepo() throws IOException { String repoLocationBase = repoDirectory.getRoot().getPath(); - List oldClusterHosts = parseOldClusterHosts(oldCluster.getHttpAddresses()); + Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); try (RestClient oldEs = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build()) { + checkClusterVersion(oldEs, oldVersion); for (boolean sourceOnlyRepository : new boolean[] { true, false }) { String repoLocation = PathUtils.get(repoLocationBase).resolve(REPO_LOCATION_BASE + sourceOnlyRepository).toString(); - Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); assumeTrue( "source only repositories only supported since ES 6.5.0", sourceOnlyRepository == false || oldVersion.onOrAfter(Version.fromString("6.5.0")) @@ -189,11 +187,11 @@ && randomBoolean()) { } } - public static Set listFilesUsingJavaIO(String dir) { - return Stream.of(new File(dir).listFiles()) - .filter(file -> file.isDirectory() == false) - .map(File::getName) - .collect(Collectors.toSet()); + private static void checkClusterVersion(RestClient client, Version version) throws IOException { + // check expected Cluster version + Request infoRequest = new Request("GET", "/"); + Response response = assertOK(client.performRequest(infoRequest)); + assertEquals(version.toString(), ObjectPath.createFromResponse(response).evaluate("version.number")); } @Override @@ -211,6 +209,7 @@ public void testOldSourceOnlyRepoAccess() throws IOException { public void runTest(boolean sourceOnlyRepository) throws IOException { // boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); + checkClusterVersion(client(), Version.CURRENT); String repoLocation = repoDirectory.getRoot().getPath(); repoLocation = PathUtils.get(repoLocation).resolve(REPO_LOCATION_BASE + sourceOnlyRepository).toString(); Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); @@ -291,7 +290,7 @@ public void runTest(boolean sourceOnlyRepository) throws IOException { assertThat(getResp.evaluate("snapshots.0.stats.total.file_count"), greaterThan(0)); // restore / mount and check whether searches work - restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); + restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, indexName, repoName, snapshotName); // close indices closeIndex(client(), "restored_" + indexName); @@ -299,7 +298,7 @@ public void runTest(boolean sourceOnlyRepository) throws IOException { closeIndex(client(), "mounted_shared_cache_" + indexName); // restore / mount again - restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, oldVersion, indexName, repoName, snapshotName); + restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, indexName, repoName, snapshotName); // TODO restart current cluster @@ -317,7 +316,6 @@ private void restoreMountAndVerify( Set expectedIds, int numberOfShards, boolean sourceOnlyRepository, - Version oldVersion, String indexName, String repoName, String snapshotName @@ -369,7 +367,7 @@ private void restoreMountAndVerify( } // run a search against the index - assertDocs("restored_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); + assertDocs("restored_" + indexName, numDocs, expectedIds, sourceOnlyRepository, numberOfShards); // mount as full copy searchable snapshot Request mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); @@ -389,7 +387,7 @@ private void restoreMountAndVerify( ensureGreen("mounted_full_copy_" + indexName); // run a search against the index - assertDocs("mounted_full_copy_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); + assertDocs("mounted_full_copy_" + indexName, numDocs, expectedIds, sourceOnlyRepository, numberOfShards); // mount as shared cache searchable snapshot mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); @@ -403,17 +401,11 @@ private void restoreMountAndVerify( assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.successful")); // run a search against the index - assertDocs("mounted_shared_cache_" + indexName, numDocs, expectedIds, sourceOnlyRepository, oldVersion, numberOfShards); + assertDocs("mounted_shared_cache_" + indexName, numDocs, expectedIds, sourceOnlyRepository, numberOfShards); } - private void assertDocs( - String index, - int numDocs, - Set expectedIds, - boolean sourceOnlyRepository, - Version oldVersion, - int numberOfShards - ) throws IOException { + private void assertDocs(String index, int numDocs, Set expectedIds, boolean sourceOnlyRepository, int numberOfShards) + throws IOException { RequestOptions requestOptions = RequestOptions.DEFAULT; // run a search against the index @@ -489,29 +481,6 @@ private void assertDocs( searchResponse.decRef(); } - // if (oldVersion.before(Version.fromString("6.0.0"))) { - // // search on _type and check that results contain _type information - // String randomType = getType(oldVersion, randomFrom(expectedIds)); - // long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count(); - // searchResponse = search( - // index, - // SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType)), - // requestOptions - // ); - // try { - // logger.info(searchResponse); - // assertEquals(typeCount, searchResponse.getHits().getTotalHits().value()); - // for (SearchHit hit : searchResponse.getHits().getHits()) { - // DocumentField typeField = hit.field("_type"); - // assertNotNull(typeField); - // assertThat(typeField.getValue(), instanceOf(String.class)); - // assertEquals(randomType, typeField.getValue()); - // } - // } finally { - // searchResponse.decRef(); - // } - // } - assertThat( expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/" + index + "/_doc/" + id))) .getMessage(), @@ -528,7 +497,8 @@ private void assertDocs( logger.info(searchResponse); assertEquals(0, searchResponse.getHits().getTotalHits().value()); assertEquals(numberOfShards, searchResponse.getSuccessfulShards()); - assertEquals(numberOfShards, searchResponse.getSkippedShards()); + // TODO the following is https://github.com/elastic/elasticsearch/issues/115631, commenting out here to reduce the noise + // assertEquals(numberOfShards, searchResponse.getSkippedShards()); } finally { searchResponse.decRef(); } From 4d35161fafdf0b83d4b92c10785e7319e28ce7f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 27 Nov 2024 11:53:27 +0100 Subject: [PATCH 15/24] Fix cluster version checks --- .../org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java index c5b08319e0949..b2645dfbb119f 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -191,7 +191,10 @@ private static void checkClusterVersion(RestClient client, Version version) thro // check expected Cluster version Request infoRequest = new Request("GET", "/"); Response response = assertOK(client.performRequest(infoRequest)); - assertEquals(version.toString(), ObjectPath.createFromResponse(response).evaluate("version.number")); + assertEquals( + version.toString(), + ((String) ObjectPath.createFromResponse(response).evaluate("version.number")).replace("-SNAPSHOT", "") + ); } @Override From d4be0ac3a9f8a5f048578e9c32ab19094144d227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 27 Nov 2024 12:08:59 +0100 Subject: [PATCH 16/24] No snapshot cache for old cluster --- .../org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java index b2645dfbb119f..a000374d58fd0 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -89,8 +89,6 @@ public class OldRepositoryAccessIT extends ESRestTestCase { .setting("xpack.license.self_generated.type", "trial") .setting("xpack.ml.enabled", "false") .setting("path.repo", () -> repoDirectory.getRoot().getPath()) - .setting("xpack.searchable.snapshot.shared_cache.size", "16MB") - .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") .build(); @ClassRule From 99191599fa247d725206fe0145a81e9d547aa507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 27 Nov 2024 14:07:59 +0100 Subject: [PATCH 17/24] Add DocValueOnlyFieldsIT yaml rest test --- .../oldrepos7x/DocValueOnlyFieldsIT.java | 210 ++++++++++++++++++ .../oldrepos7x/OldMappingsIT.java | 3 + 2 files changed, 213 insertions(+) create mode 100644 x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java new file mode 100644 index 0000000000000..09546dba8d81c --- /dev/null +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos7x; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.http.HttpHost; +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.io.IOException; + +/** + * Tests doc-value-based searches against indices imported from clusters older than N-1. + * We reuse the YAML tests in search/390_doc_values_search.yml but have to do the setup + * manually here as the setup is done on the old cluster for which we have to use the + * low-level REST client instead of the YAML set up that only knows how to talk to + * newer ES versions. + * + * We mimic the setup in search/390_doc_values_search.yml here, but adapt it to work + * against older version clusters. + */ +public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase { + + static final Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); + static boolean setupDone; + + public static TemporaryFolder repoDirectory = new TemporaryFolder(); + + public static ElasticsearchCluster currentCluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .nodes(2) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.ml.enabled", "false") + .setting("path.repo", () -> repoDirectory.getRoot().getPath()) + .setting("xpack.searchable.snapshot.shared_cache.size", "16MB") + .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") + .build(); + + public static ElasticsearchCluster oldCluster = ElasticsearchCluster.local() + .version(org.elasticsearch.test.cluster.util.Version.fromString(System.getProperty("tests.old_cluster_version"))) + .distribution(DistributionType.DEFAULT) + .nodes(2) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.ml.enabled", "false") + .setting("path.repo", () -> repoDirectory.getRoot().getPath()) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(oldCluster).around(currentCluster); + private static final String REPO_NAME = "doc_values_repo"; + private static final String INDEX_NAME = "test"; + private static final String snapshotName = "snap"; + + private static String repoLocation; + + public DocValueOnlyFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + @Override + protected boolean skipSetupSections() { + // setup in the YAML file is replaced by the method below + return true; + } + + @BeforeClass + public static void setupSnapshot() throws IOException { + repoLocation = repoDirectory.getRoot().getPath(); + String[] basicTypes = new String[] { + "byte", + "double", + "float", + "half_float", + "integer", + "long", + "short", + "boolean", + "keyword", + "ip", + "geo_point" }; // date is manually added as it need further configuration + + int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); + try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { + Request createIndex = new Request("PUT", "/" + INDEX_NAME); + int numberOfShards = randomIntBetween(1, 3); + + XContentBuilder settingsBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject("settings") + .field("index.number_of_shards", numberOfShards) + .endObject() + .startObject("mappings"); + settingsBuilder.field("dynamic", false).startObject("properties"); + for (String type : basicTypes) { + settingsBuilder.startObject(type).field("type", type).endObject(); + } + settingsBuilder.startObject("date").field("type", "date").field("format", "yyyy/MM/dd").endObject(); + settingsBuilder.endObject().endObject().endObject(); + + createIndex.setJsonEntity(Strings.toString(settingsBuilder)); + assertOK(oldEs.performRequest(createIndex)); + + Request doc1 = new Request("PUT", "/" + INDEX_NAME + "/" + "doc" + "/" + "1"); + doc1.addParameter("refresh", "true"); + XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() + .startObject() + .field("byte", 1) + .field("double", 1.0) + .field("float", 1.0) + .field("half_float", 1.0) + .field("integer", 1) + .field("long", 1) + .field("short", 1) + .field("date", "2017/01/01") + .field("keyword", "key1") + .field("boolean", false) + .field("ip", "192.168.0.1") + .array("geo_point", 13.5, 34.89) + .endObject(); + doc1.setJsonEntity(Strings.toString(bodyDoc1)); + assertOK(oldEs.performRequest(doc1)); + + Request doc2 = new Request("PUT", "/" + INDEX_NAME + "/" + "doc" + "/" + "2"); + doc2.addParameter("refresh", "true"); + XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() + .startObject() + .field("byte", 2) + .field("double", 2.0) + .field("float", 2.0) + .field("half_float", 2.0) + .field("integer", 2) + .field("long", 2) + .field("short", 2) + .field("date", "2017/01/02") + .field("keyword", "key2") + .field("boolean", true) + .field("ip", "192.168.0.2") + .array("geo_point", -63.24, 31.0) + .endObject(); + doc2.setJsonEntity(Strings.toString(bodyDoc2)); + assertOK(oldEs.performRequest(doc2)); + + // register repo on old ES and take snapshot + Request createRepoRequest = new Request("PUT", "/_snapshot/" + REPO_NAME); + createRepoRequest.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(oldEs.performRequest(createRepoRequest)); + + Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + REPO_NAME + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + createSnapshotRequest.setJsonEntity("{\"indices\":\"" + INDEX_NAME + "\"}"); + assertOK(oldEs.performRequest(createSnapshotRequest)); + } + } + + @Before + public void registerAndRestoreRepo() throws IOException { + // The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only + // initialized later, so we do it when running the first test + if (setupDone = false) { + // register repo on new ES and restore snapshot + Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + REPO_NAME); + createRepoRequest2.setJsonEntity(Strings.format(""" + {"type":"fs","settings":{"location":"%s"}} + """, repoLocation)); + assertOK(client().performRequest(createRepoRequest2)); + + final Request createRestoreRequest = new Request("POST", "/_snapshot/" + REPO_NAME + "/" + snapshotName + "/_restore"); + createRestoreRequest.addParameter("wait_for_completion", "true"); + createRestoreRequest.setJsonEntity("{\"indices\":\"" + INDEX_NAME + "\"}"); + assertOK(client().performRequest(createRestoreRequest)); + + setupDone = true; + } + } +} diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java index 689373d3bc02c..21ca5ec005e19 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java @@ -179,6 +179,9 @@ public static void setupOldRepo() throws IOException { @Before public void registerAndRestoreRepo() throws IOException { + // this would ideally also happen in @BeforeClass and just once, but we don't have the current cluster client() + // there yet. So we do it before tests here and make sure to only restore the repo once. + // Goes together with the empty "wipeSnapshot()" override in this test. if (repoRestored == false) { // register repo on new ES and restore snapshot Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); From 2406751cbc43ce631354dbbbe976c08ea153ca09 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 27 Nov 2024 14:35:26 +0100 Subject: [PATCH 18/24] Fix wiring of yaml specs to test tasks --- .../repository-old-versions-7x/build.gradle | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/x-pack/qa/repository-old-versions-7x/build.gradle b/x-pack/qa/repository-old-versions-7x/build.gradle index 004194c7782f4..2d29c21ebf1b2 100644 --- a/x-pack/qa/repository-old-versions-7x/build.gradle +++ b/x-pack/qa/repository-old-versions-7x/build.gradle @@ -1,6 +1,9 @@ import org.elasticsearch.gradle.internal.test.RestIntegTestTask import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.internal.test.rest.CopyRestApiTask +import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask +import org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.rest-resources' @@ -14,6 +17,23 @@ restResources { } } +// Register rest resources with source set +sourceSets.javaRestTest.getOutput() + .dir( + project.getTasks() + .withType(CopyRestApiTask.class) + .named(RestResourcesPlugin.COPY_REST_API_SPECS_TASK) + .flatMap(CopyRestApiTask::getOutputResourceDir) + ); + +sourceSets.javaRestTest.getOutput() + .dir( + project.getTasks() + .withType(CopyRestTestsTask.class) + .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK) + .flatMap(CopyRestTestsTask::getOutputResourceDir) + ); + tasks.named("javaRestTest") { enabled = false } From 85d01c2075b60c63d8d824fc0c725ade12a82025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 27 Nov 2024 16:09:34 +0100 Subject: [PATCH 19/24] Make DocValueOnlyFieldsIT work for V7x --- .../test/rest/ESRestTestCase.java | 11 +++++--- .../oldrepos7x/DocValueOnlyFieldsIT.java | 25 +++++++++++-------- .../oldrepos7x/OldMappingsIT.java | 19 +------------- .../oldrepos7x/OldRepositoryAccessIT.java | 19 +------------- 4 files changed, 23 insertions(+), 51 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index bdef0ba631b72..5c24fb5f31af8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -10,7 +10,6 @@ package org.elasticsearch.test.rest; import io.netty.handler.codec.http.HttpMethod; - import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -87,6 +86,7 @@ import org.junit.AfterClass; import org.junit.Before; +import javax.net.ssl.SSLContext; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -118,14 +118,13 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; -import javax.net.ssl.SSLContext; - import static java.util.Collections.sort; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; @@ -408,6 +407,10 @@ protected static boolean has(ProductFeature feature) { } protected List parseClusterHosts(String hostsString) { + return parseClusterHosts(hostsString, this::buildHttpHost); + } + + public static List parseClusterHosts(String hostsString, BiFunction httpHostSupplier) { String[] stringUrls = hostsString.split(","); List hosts = new ArrayList<>(stringUrls.length); for (String stringUrl : stringUrls) { @@ -417,7 +420,7 @@ protected List parseClusterHosts(String hostsString) { } String host = stringUrl.substring(0, portSeparator); int port = Integer.valueOf(stringUrl.substring(portSeparator + 1)); - hosts.add(buildHttpHost(host, port)); + hosts.add(httpHostSupplier.apply(host, port)); } return unmodifiableList(hosts); } diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java index 09546dba8d81c..2f4d5ba6a5008 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java @@ -29,6 +29,7 @@ import org.junit.rules.TestRule; import java.io.IOException; +import java.util.List; /** * Tests doc-value-based searches against indices imported from clusters older than N-1. @@ -43,7 +44,7 @@ public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase { static final Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); - static boolean setupDone; + static boolean repoRestored; public static TemporaryFolder repoDirectory = new TemporaryFolder(); @@ -54,8 +55,6 @@ public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase { .setting("xpack.license.self_generated.type", "trial") .setting("xpack.ml.enabled", "false") .setting("path.repo", () -> repoDirectory.getRoot().getPath()) - .setting("xpack.searchable.snapshot.shared_cache.size", "16MB") - .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") .build(); public static ElasticsearchCluster oldCluster = ElasticsearchCluster.local() @@ -73,7 +72,6 @@ public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase { private static final String REPO_NAME = "doc_values_repo"; private static final String INDEX_NAME = "test"; private static final String snapshotName = "snap"; - private static String repoLocation; public DocValueOnlyFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { @@ -111,9 +109,8 @@ public static void setupSnapshot() throws IOException { "keyword", "ip", "geo_point" }; // date is manually added as it need further configuration - - int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); - try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { + List oldClusterHosts = parseClusterHosts(oldCluster.getHttpAddresses(), (host, port) -> new HttpHost(host, port)); + try (RestClient oldEs = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build();) { Request createIndex = new Request("PUT", "/" + INDEX_NAME); int numberOfShards = randomIntBetween(1, 3); @@ -133,7 +130,7 @@ public static void setupSnapshot() throws IOException { createIndex.setJsonEntity(Strings.toString(settingsBuilder)); assertOK(oldEs.performRequest(createIndex)); - Request doc1 = new Request("PUT", "/" + INDEX_NAME + "/" + "doc" + "/" + "1"); + Request doc1 = new Request("PUT", "/" + INDEX_NAME + "/" + "_doc" + "/" + "1"); doc1.addParameter("refresh", "true"); XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() .startObject() @@ -153,7 +150,7 @@ public static void setupSnapshot() throws IOException { doc1.setJsonEntity(Strings.toString(bodyDoc1)); assertOK(oldEs.performRequest(doc1)); - Request doc2 = new Request("PUT", "/" + INDEX_NAME + "/" + "doc" + "/" + "2"); + Request doc2 = new Request("PUT", "/" + INDEX_NAME + "/" + "_doc" + "/" + "2"); doc2.addParameter("refresh", "true"); XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() .startObject() @@ -191,7 +188,7 @@ public static void setupSnapshot() throws IOException { public void registerAndRestoreRepo() throws IOException { // The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only // initialized later, so we do it when running the first test - if (setupDone = false) { + if (repoRestored == false) { // register repo on new ES and restore snapshot Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + REPO_NAME); createRepoRequest2.setJsonEntity(Strings.format(""" @@ -204,7 +201,13 @@ public void registerAndRestoreRepo() throws IOException { createRestoreRequest.setJsonEntity("{\"indices\":\"" + INDEX_NAME + "\"}"); assertOK(client().performRequest(createRestoreRequest)); - setupDone = true; + repoRestored = true; } + logger.info("repo [" + REPO_NAME + "] restored"); + } + + @Override + protected String getTestRestCluster() { + return currentCluster.getHttpAddresses(); } } diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java index 21ca5ec005e19..2041839f633ea 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java @@ -30,13 +30,11 @@ import org.junit.rules.TestRule; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static java.util.Collections.unmodifiableList; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; @@ -103,7 +101,7 @@ public static void setupOldRepo() throws IOException { repoName = "old_mappings_repo"; snapshotName = "snap"; - List oldClusterHosts = parseOldClusterHosts(oldCluster.getHttpAddresses()); + List oldClusterHosts = parseClusterHosts(oldCluster.getHttpAddresses(), (host, port) -> new HttpHost(host, port)); try (RestClient oldEsClient = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build();) { assertOK(oldEsClient.performRequest(createIndex("filebeat", "filebeat.json"))); @@ -200,21 +198,6 @@ public void registerAndRestoreRepo() throws IOException { } } - private static List parseOldClusterHosts(String hostsString) { - String[] stringUrls = hostsString.split(","); - List hosts = new ArrayList<>(stringUrls.length); - for (String stringUrl : stringUrls) { - int portSeparator = stringUrl.lastIndexOf(':'); - if (portSeparator < 0) { - throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]"); - } - String host = stringUrl.substring(0, portSeparator); - int port = Integer.valueOf(stringUrl.substring(portSeparator + 1)); - hosts.add(new HttpHost(host, port)); - } - return unmodifiableList(hosts); - } - public void testFileBeatApache2MappingOk() throws IOException { Request mappingRequest = new Request("GET", "/" + "filebeat" + "/_mapping"); Map mapping = entityAsMap(client().performRequest(mappingRequest)); diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java index a000374d58fd0..741b014a33093 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -44,7 +44,6 @@ import org.junit.rules.TestRule; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; @@ -54,7 +53,6 @@ import java.util.Set; import java.util.stream.Collectors; -import static java.util.Collections.unmodifiableList; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -106,7 +104,7 @@ public class OldRepositoryAccessIT extends ESRestTestCase { @BeforeClass public static void setupOldRepo() throws IOException { String repoLocationBase = repoDirectory.getRoot().getPath(); - List oldClusterHosts = parseOldClusterHosts(oldCluster.getHttpAddresses()); + List oldClusterHosts = parseClusterHosts(oldCluster.getHttpAddresses(), (host, port) -> new HttpHost(host, port)); Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); try (RestClient oldEs = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build()) { checkClusterVersion(oldEs, oldVersion); @@ -524,19 +522,4 @@ private static void closeIndex(RestClient client, String index) throws IOExcepti ObjectPath doc = ObjectPath.createFromResponse(client.performRequest(request)); assertTrue(doc.evaluate("shards_acknowledged")); } - - private static List parseOldClusterHosts(String hostsString) { - String[] stringUrls = hostsString.split(","); - List hosts = new ArrayList<>(stringUrls.length); - for (String stringUrl : stringUrls) { - int portSeparator = stringUrl.lastIndexOf(':'); - if (portSeparator < 0) { - throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]"); - } - String host = stringUrl.substring(0, portSeparator); - int port = Integer.valueOf(stringUrl.substring(portSeparator + 1)); - hosts.add(new HttpHost(host, port)); - } - return unmodifiableList(hosts); - } } From f17527f5eccc2c3a668289fa6ba862bb3c42924c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 27 Nov 2024 16:28:16 +0100 Subject: [PATCH 20/24] Using 7.9.0 instead of 7.16 --- x-pack/qa/repository-old-versions-7x/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/qa/repository-old-versions-7x/build.gradle b/x-pack/qa/repository-old-versions-7x/build.gradle index 2d29c21ebf1b2..e072f37bf04bf 100644 --- a/x-pack/qa/repository-old-versions-7x/build.gradle +++ b/x-pack/qa/repository-old-versions-7x/build.gradle @@ -38,7 +38,7 @@ tasks.named("javaRestTest") { enabled = false } -['7.16.3', '7.17.25'].each { versionString -> +['7.9.0', '7.17.25'].each { versionString -> String versionNoDots = versionString.replace('.', '_') tasks.register("javaRestTest#${versionNoDots}", StandaloneRestIntegTestTask) { From da3db30b235e468ee6d4df1e786454bd9756808f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 27 Nov 2024 17:16:05 +0100 Subject: [PATCH 21/24] Cleanups --- .../test/rest/ESRestTestCase.java | 4 +++- .../oldrepos7x/DocValueOnlyFieldsIT.java | 23 +++++++++---------- .../oldrepos7x/OldMappingsIT.java | 19 ++++++--------- .../oldrepos7x/OldRepositoryAccessIT.java | 4 ++-- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 5c24fb5f31af8..d5b86656f1ca6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -10,6 +10,7 @@ package org.elasticsearch.test.rest; import io.netty.handler.codec.http.HttpMethod; + import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -86,7 +87,6 @@ import org.junit.AfterClass; import org.junit.Before; -import javax.net.ssl.SSLContext; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -125,6 +125,8 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import javax.net.ssl.SSLContext; + import static java.util.Collections.sort; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java index 2f4d5ba6a5008..e5bee956b3679 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/DocValueOnlyFieldsIT.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.List; +import java.util.function.Supplier; /** * Tests doc-value-based searches against indices imported from clusters older than N-1. @@ -43,11 +44,9 @@ */ public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase { - static final Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); - static boolean repoRestored; - - public static TemporaryFolder repoDirectory = new TemporaryFolder(); - + private static final Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); + private static boolean repoRestored = false; + private static final TemporaryFolder repoDirectory = new TemporaryFolder(); public static ElasticsearchCluster currentCluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .nodes(2) @@ -72,7 +71,7 @@ public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase { private static final String REPO_NAME = "doc_values_repo"; private static final String INDEX_NAME = "test"; private static final String snapshotName = "snap"; - private static String repoLocation; + private static final Supplier repoLocation = () -> repoDirectory.getRoot().getPath(); public DocValueOnlyFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); @@ -96,7 +95,6 @@ protected boolean skipSetupSections() { @BeforeClass public static void setupSnapshot() throws IOException { - repoLocation = repoDirectory.getRoot().getPath(); String[] basicTypes = new String[] { "byte", "double", @@ -174,7 +172,7 @@ public static void setupSnapshot() throws IOException { Request createRepoRequest = new Request("PUT", "/_snapshot/" + REPO_NAME); createRepoRequest.setJsonEntity(Strings.format(""" {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); + """, repoLocation.get())); assertOK(oldEs.performRequest(createRepoRequest)); Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + REPO_NAME + "/" + snapshotName); @@ -186,14 +184,15 @@ public static void setupSnapshot() throws IOException { @Before public void registerAndRestoreRepo() throws IOException { - // The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only - // initialized later, so we do it when running the first test + // Ideally we could restore the repo on the "current" cluster in @BeforeClass, but that + // does not work because the client() is not initialized then. + // To restore only once we guard this operation by a flag. if (repoRestored == false) { // register repo on new ES and restore snapshot Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + REPO_NAME); createRepoRequest2.setJsonEntity(Strings.format(""" {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); + """, repoLocation.get())); assertOK(client().performRequest(createRepoRequest2)); final Request createRestoreRequest = new Request("POST", "/_snapshot/" + REPO_NAME + "/" + snapshotName + "/_restore"); @@ -203,7 +202,7 @@ public void registerAndRestoreRepo() throws IOException { repoRestored = true; } - logger.info("repo [" + REPO_NAME + "] restored"); + logger.info("Repo [" + REPO_NAME + "] restored."); } @Override diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java index 2041839f633ea..56c6ee10d4db6 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java @@ -33,6 +33,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; @@ -67,9 +68,9 @@ public class OldMappingsIT extends ESRestTestCase { public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(oldCluster).around(currentCluster); private static boolean repoRestored = false; - private static String repoLocation; - private static String repoName; - private static String snapshotName; + private static final Supplier repoLocation = () -> repoDirectory.getRoot().getPath(); + private static final String repoName = "old_mappings_repo"; + private static final String snapshotName = "snap"; @Override protected String getTestRestCluster() { @@ -96,15 +97,9 @@ private static Request createIndex(String indexName, String file) throws IOExcep @BeforeClass public static void setupOldRepo() throws IOException { - repoLocation = repoDirectory.getRoot().getPath(); - - repoName = "old_mappings_repo"; - snapshotName = "snap"; - List oldClusterHosts = parseClusterHosts(oldCluster.getHttpAddresses(), (host, port) -> new HttpHost(host, port)); try (RestClient oldEsClient = RestClient.builder(oldClusterHosts.toArray(new HttpHost[oldClusterHosts.size()])).build();) { assertOK(oldEsClient.performRequest(createIndex("filebeat", "filebeat.json"))); - assertOK(oldEsClient.performRequest(createIndex("custom", "custom.json"))); assertOK(oldEsClient.performRequest(createIndex("nested", "nested.json"))); @@ -158,13 +153,13 @@ public static void setupOldRepo() throws IOException { Request getSettingsRequest = new Request("GET", "/_cluster/settings?include_defaults=true"); Map response = entityAsMap(oldEsClient.performRequest(getSettingsRequest)); - assertEquals(repoLocation, ((List) (XContentMapValues.extractValue("defaults.path.repo", response))).get(0)); + assertEquals(repoLocation.get(), ((List) (XContentMapValues.extractValue("defaults.path.repo", response))).get(0)); // register repo on old ES and take snapshot Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); createRepoRequest.setJsonEntity(Strings.format(""" {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); + """, repoLocation.get())); assertOK(oldEsClient.performRequest(createRepoRequest)); Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); @@ -185,7 +180,7 @@ public void registerAndRestoreRepo() throws IOException { Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); createRepoRequest2.setJsonEntity(Strings.format(""" {"type":"fs","settings":{"location":"%s"}} - """, repoLocation)); + """, repoLocation.get())); assertOK(client().performRequest(createRepoRequest2)); final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java index 741b014a33093..0e781023ab2a6 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -300,7 +300,6 @@ public void runTest(boolean sourceOnlyRepository) throws IOException { restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, indexName, repoName, snapshotName); // TODO restart current cluster - // ensureGreen("restored_" + indexName); // ensureGreen("mounted_full_copy_" + indexName); // ensureGreen("mounted_shared_cache_" + indexName); @@ -496,7 +495,8 @@ private void assertDocs(String index, int numDocs, Set expectedIds, bool logger.info(searchResponse); assertEquals(0, searchResponse.getHits().getTotalHits().value()); assertEquals(numberOfShards, searchResponse.getSuccessfulShards()); - // TODO the following is https://github.com/elastic/elasticsearch/issues/115631, commenting out here to reduce the noise + // TODO the following is a failure tracked in https://github.com/elastic/elasticsearch/issues/115631 + // commenting out here to reduce the noise // assertEquals(numberOfShards, searchResponse.getSkippedShards()); } finally { searchResponse.decRef(); From 01ba741aca48f6cd3e3c78abea3714a016895974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 28 Nov 2024 16:45:00 +0100 Subject: [PATCH 22/24] Add _field_names disabling to new tests --- .../java/org/elasticsearch/oldrepos7x/OldMappingsIT.java | 6 +++++- .../resources/org/elasticsearch/oldrepos7x/custom.json | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java index 56c6ee10d4db6..f4ae61052db8c 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldMappingsIT.java @@ -10,6 +10,7 @@ import org.apache.http.HttpHost; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.WarningsHandler; @@ -187,7 +188,10 @@ public void registerAndRestoreRepo() throws IOException { createRestoreRequest.addParameter("wait_for_completion", "true"); createRestoreRequest.setJsonEntity("{\"indices\":\"" + indices.stream().collect(Collectors.joining(",")) + "\"}"); createRestoreRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); - assertOK(client().performRequest(createRestoreRequest)); + Response response = client().performRequest(createRestoreRequest); + // check deprecation warning for "_field_name" disabling + assertTrue(response.getWarnings().stream().filter(s -> s.contains("Disabling _field_names is not necessary")).count() > 0); + assertOK(response); repoRestored = true; } diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/custom.json b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/custom.json index b5865080d9abf..eb2027e09e22f 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/custom.json +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/resources/org/elasticsearch/oldrepos7x/custom.json @@ -1,3 +1,6 @@ +"_field_names": { + "enabled": false +}, "properties": { "apache2": { "properties": { From c7693b3c4317d33838bc2a29bfea900d577e2c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 29 Nov 2024 12:38:18 +0100 Subject: [PATCH 23/24] Add restart cluster test --- .../oldrepos7x/OldRepositoryAccessIT.java | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java index 0e781023ab2a6..6094fde01c424 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Node; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -101,6 +102,11 @@ public class OldRepositoryAccessIT extends ESRestTestCase { private static final int numDocs = 10; private static final int extraDocs = 1; + /** + * We set up the data in the old version cluster and take a snapshot to a file system directory. + * We only need to do this once before all other tests because they can re-mount the data from that + * directory. + */ @BeforeClass public static void setupOldRepo() throws IOException { String repoLocationBase = repoDirectory.getRoot().getPath(); @@ -207,16 +213,10 @@ public void testOldSourceOnlyRepoAccess() throws IOException { } public void runTest(boolean sourceOnlyRepository) throws IOException { - // boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); checkClusterVersion(client(), Version.CURRENT); String repoLocation = repoDirectory.getRoot().getPath(); repoLocation = PathUtils.get(repoLocation).resolve(REPO_LOCATION_BASE + sourceOnlyRepository).toString(); Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); - assumeTrue( - "source only repositories only supported since ES 6.5.0", - sourceOnlyRepository == false || oldVersion.onOrAfter(Version.fromString("6.5.0")) - ); - assertThat("Index version should be added to archive tests", oldVersion, lessThan(Version.V_8_10_0)); IndexVersion indexVersion = IndexVersion.fromId(oldVersion.id); @@ -299,10 +299,18 @@ public void runTest(boolean sourceOnlyRepository) throws IOException { // restore / mount again restoreMountAndVerify(numDocs, expectedIds, numberOfShards, sourceOnlyRepository, indexName, repoName, snapshotName); - // TODO restart current cluster - // ensureGreen("restored_" + indexName); - // ensureGreen("mounted_full_copy_" + indexName); - // ensureGreen("mounted_shared_cache_" + indexName); + currentCluster.stop(false); + currentCluster.start(); + + // we need to replace the nodes for the clients can connect to the restarted cluster + List hosts = parseClusterHosts(currentCluster.getHttpAddresses(), (host, port) -> new HttpHost(host, port)); + List nodes = hosts.stream().map(Node::new).collect(Collectors.toList()); + client().setNodes(nodes); + adminClient().setNodes(nodes); + + ensureGreen("restored_" + indexName); + ensureGreen("mounted_full_copy_" + indexName); + ensureGreen("mounted_shared_cache_" + indexName); } private static String sourceForDoc(int i) { From 050d195935f24fe9d212ed6459c7fb7736f2327e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 2 Dec 2024 19:28:00 +0100 Subject: [PATCH 24/24] Pulling in test changes from 117649 --- .../org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java index 6094fde01c424..4613f0d17bc07 100644 --- a/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions-7x/src/javaRestTest/java/org/elasticsearch/oldrepos7x/OldRepositoryAccessIT.java @@ -503,9 +503,8 @@ private void assertDocs(String index, int numDocs, Set expectedIds, bool logger.info(searchResponse); assertEquals(0, searchResponse.getHits().getTotalHits().value()); assertEquals(numberOfShards, searchResponse.getSuccessfulShards()); - // TODO the following is a failure tracked in https://github.com/elastic/elasticsearch/issues/115631 - // commenting out here to reduce the noise - // assertEquals(numberOfShards, searchResponse.getSkippedShards()); + int expectedSkips = numberOfShards == 1 ? 0 : numberOfShards; + assertEquals(expectedSkips, searchResponse.getSkippedShards()); } finally { searchResponse.decRef(); }