From e34601dbb8d803495e0e3b537ea9ac10f0517ccf Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 31 Mar 2022 15:41:59 +0200 Subject: [PATCH] Remove archival functionality from 8.2 branch (#85524) Archival won't ship with 8.2, hence removing the files there. I found it easier to remove the code than reintroducing a way to only enable it in snapshot builds. Relates #81210 --- .../xcontent/monitor_cluster_stats.json | 5 - docs/reference/rest-api/info.asciidoc | 4 - docs/reference/rest-api/usage.asciidoc | 5 - .../xpack/core/XPackClientPlugin.java | 5 +- .../elasticsearch/xpack/core/XPackField.java | 2 - .../core/action/XPackInfoFeatureAction.java | 4 +- .../core/action/XPackUsageFeatureAction.java | 4 +- .../core/archive/ArchiveFeatureSetUsage.java | 73 - .../archive/ArchiveFeatureSetUsageTests.java | 39 - .../plugin/old-lucene-versions/build.gradle | 18 - .../lucene/bwc/AbstractArchiveTestCase.java | 113 - .../lucene/bwc/ArchiveLicenseIntegTests.java | 146 -- .../ArchiveSettingValidationIntegTests.java | 49 - .../bwc/LocalStateOldLuceneVersions.java | 32 - .../lucene/bwc/ArchiveAllocationDecider.java | 60 - .../bwc/ArchiveInfoTransportAction.java | 44 - .../xpack/lucene/bwc/ArchiveUsageTracker.java | 43 - .../bwc/ArchiveUsageTransportAction.java | 70 - ...ShardsOnInvalidLicenseClusterListener.java | 98 - .../xpack/lucene/bwc/OldLuceneVersions.java | 229 -- .../xpack/lucene/bwc/OldSegmentInfos.java | 737 ------- .../xpack/lucene/bwc/codecs/BWCCodec.java | 212 -- .../codecs/index/LegacyBinaryDocValues.java | 42 - .../index/LegacyBinaryDocValuesWrapper.java | 93 - .../index/LegacyDocValuesIterables.java | 539 ----- .../codecs/index/LegacyNumericDocValues.java | 42 - .../index/LegacyNumericDocValuesWrapper.java | 99 - .../codecs/index/LegacySortedDocValues.java | 114 - .../index/LegacySortedDocValuesWrapper.java | 104 - .../index/LegacySortedNumericDocValues.java | 53 - .../LegacySortedNumericDocValuesWrapper.java | 102 - .../index/LegacySortedSetDocValues.java | 115 - .../LegacySortedSetDocValuesWrapper.java | 115 - .../lucene50/Lucene50FieldInfosFormat.java | 171 -- .../lucene50/Lucene50SegmentInfoFormat.java | 95 - .../codecs/lucene54/LegacyStringHelper.java | 72 - .../lucene54/Lucene54DocValuesConsumer.java | 842 -------- .../lucene54/Lucene54DocValuesFormat.java | 119 -- .../lucene54/Lucene54DocValuesProducer.java | 1847 ----------------- .../bwc/codecs/lucene60/Lucene60Codec.java | 107 - .../bwc/codecs/lucene62/Lucene62Codec.java | 96 - .../lucene62/Lucene62SegmentInfoFormat.java | 240 --- .../bwc/codecs/lucene70/BWCLucene70Codec.java | 75 - .../services/org.apache.lucene.codecs.Codec | 10 - .../org.apache.lucene.codecs.DocValuesFormat | 16 - .../Lucene54DocValuesFormatTests.java | 22 - .../xpack/security/operator/Constants.java | 2 - .../qa/repository-old-versions/build.gradle | 165 -- .../oldrepos/DocValueOnlyFieldsIT.java | 213 -- .../oldrepos/OldRepositoryAccessIT.java | 499 ----- 50 files changed, 3 insertions(+), 7998 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsage.java delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsageTests.java delete mode 100644 x-pack/plugin/old-lucene-versions/build.gradle delete mode 100644 x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveSettingValidationIntegTests.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/LocalStateOldLuceneVersions.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveAllocationDecider.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveInfoTransportAction.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTracker.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTransportAction.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/FailShardsOnInvalidLicenseClusterListener.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValues.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValuesWrapper.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyDocValuesIterables.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyNumericDocValues.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyNumericDocValuesWrapper.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedDocValues.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedDocValuesWrapper.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedNumericDocValues.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedNumericDocValuesWrapper.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValues.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/LegacyStringHelper.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesConsumer.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormat.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesProducer.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec delete mode 100644 x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat delete mode 100644 x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormatTests.java delete mode 100644 x-pack/qa/repository-old-versions/build.gradle delete mode 100644 x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java delete mode 100644 x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java diff --git a/benchmarks/src/main/resources/org/elasticsearch/benchmark/xcontent/monitor_cluster_stats.json b/benchmarks/src/main/resources/org/elasticsearch/benchmark/xcontent/monitor_cluster_stats.json index eea13dec75ffd..de460d770d249 100644 --- a/benchmarks/src/main/resources/org/elasticsearch/benchmark/xcontent/monitor_cluster_stats.json +++ b/benchmarks/src/main/resources/org/elasticsearch/benchmark/xcontent/monitor_cluster_stats.json @@ -1233,11 +1233,6 @@ "total" : 0, "failed" : 0 } - }, - "archive" : { - "available" : false, - "enabled" : true, - "indices_count" : 0 } } } diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index 5292b6e8967cb..e4d533c8378d3 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -81,10 +81,6 @@ Example response: "available" : true, "enabled" : true }, - "archive" : { - "available" : true, - "enabled" : true - }, "enrich" : { "available" : true, "enabled" : true diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 13773b02fe417..786a21f576423 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -395,11 +395,6 @@ GET /_xpack/usage "aggregate_metric" : { "available" : true, "enabled" : true - }, - "archive" : { - "available" : true, - "enabled" : true, - "indices_count" : 0 } } ------------------------------------------------------------ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 68a506d1157ec..db0ac2aebafa9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -36,7 +36,6 @@ import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.aggregatemetric.AggregateMetricFeatureSetUsage; import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; -import org.elasticsearch.xpack.core.archive.ArchiveFeatureSetUsage; import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage; @@ -559,9 +558,7 @@ public List getNamedWriteables() { // Data Streams new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_STREAMS, DataStreamFeatureSetUsage::new), // Data Tiers - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_TIERS, DataTiersFeatureSetUsage::new), - // Archive - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ARCHIVE, ArchiveFeatureSetUsage::new) + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_TIERS, DataTiersFeatureSetUsage::new) ) ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index dbc100e62ac1e..59343705b9098 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -71,8 +71,6 @@ public final class XPackField { public static final String AGGREGATE_METRIC = "aggregate_metric"; /** Name constant for the operator privileges feature. */ public static final String OPERATOR_PRIVILEGES = "operator_privileges"; - /** Name constant for the archive feature. */ - public static final String ARCHIVE = "archive"; private XPackField() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java index 83e835d4bb6dc..c6c941ef3092d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java @@ -47,7 +47,6 @@ public class XPackInfoFeatureAction extends ActionType public static final XPackInfoFeatureAction DATA_STREAMS = new XPackInfoFeatureAction(XPackField.DATA_STREAMS); public static final XPackInfoFeatureAction DATA_TIERS = new XPackInfoFeatureAction(XPackField.DATA_TIERS); public static final XPackInfoFeatureAction AGGREGATE_METRIC = new XPackInfoFeatureAction(XPackField.AGGREGATE_METRIC); - public static final XPackInfoFeatureAction ARCHIVE = new XPackInfoFeatureAction(XPackField.ARCHIVE); public static final List ALL; static { @@ -75,8 +74,7 @@ public class XPackInfoFeatureAction extends ActionType DATA_STREAMS, SEARCHABLE_SNAPSHOTS, DATA_TIERS, - AGGREGATE_METRIC, - ARCHIVE + AGGREGATE_METRIC ) ); ALL = Collections.unmodifiableList(actions); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index bfbac109012e5..cd310064ffa0f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -44,7 +44,6 @@ public class XPackUsageFeatureAction extends ActionType ALL = List.of( AGGREGATE_METRIC, @@ -67,8 +66,7 @@ public class XPackUsageFeatureAction extends ActionType { - - @Override - protected ArchiveFeatureSetUsage createTestInstance() { - boolean available = randomBoolean(); - return new ArchiveFeatureSetUsage(available, randomIntBetween(0, 100000)); - } - - @Override - protected ArchiveFeatureSetUsage mutateInstance(ArchiveFeatureSetUsage instance) throws IOException { - boolean available = instance.available(); - int numArchiveIndices = instance.getNumberOfArchiveIndices(); - switch (between(0, 1)) { - case 0 -> available = available == false; - case 1 -> numArchiveIndices = randomValueOtherThan(numArchiveIndices, () -> randomIntBetween(0, 100000)); - default -> throw new AssertionError("Illegal randomisation branch"); - } - return new ArchiveFeatureSetUsage(available, numArchiveIndices); - } - - @Override - protected Writeable.Reader instanceReader() { - return ArchiveFeatureSetUsage::new; - } - -} diff --git a/x-pack/plugin/old-lucene-versions/build.gradle b/x-pack/plugin/old-lucene-versions/build.gradle deleted file mode 100644 index e59b68f040f6f..0000000000000 --- a/x-pack/plugin/old-lucene-versions/build.gradle +++ /dev/null @@ -1,18 +0,0 @@ -apply plugin: 'elasticsearch.internal-cluster-test' -apply plugin: 'elasticsearch.internal-es-plugin' -apply plugin: 'elasticsearch.internal-test-artifact' - -esplugin { - name 'old-lucene-versions' - description 'A plugin for accessing older Lucene indices' - classname 'org.elasticsearch.xpack.lucene.bwc.OldLuceneVersions' - extendedPlugins = ['x-pack-core'] -} -archivesBaseName = 'x-pack-old-lucene-versions' - -dependencies { - compileOnly project(path: xpackModule('core')) - internalClusterTestImplementation(testArtifact(project(xpackModule('core')))) -} - -addQaCheckDependencies() diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java deleted file mode 100644 index dc877b2d90cb5..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.RepositoryMetadata; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.env.Environment; -import org.elasticsearch.indices.recovery.RecoverySettings; -import org.elasticsearch.license.License; -import org.elasticsearch.license.PostStartTrialAction; -import org.elasticsearch.license.PostStartTrialRequest; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.repositories.IndexId; -import org.elasticsearch.repositories.Repository; -import org.elasticsearch.repositories.RepositoryData; -import org.elasticsearch.repositories.fs.FsRepository; -import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; -import org.elasticsearch.snapshots.SnapshotId; -import org.elasticsearch.snapshots.mockstore.MockRepository; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.junit.Before; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; - -@ESIntegTestCase.ClusterScope(supportsDedicatedMasters = false, numClientNodes = 0, scope = ESIntegTestCase.Scope.TEST) -public abstract class AbstractArchiveTestCase extends AbstractSnapshotIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(LocalStateOldLuceneVersions.class, TestRepositoryPlugin.class, MockRepository.Plugin.class); - } - - public static class TestRepositoryPlugin extends Plugin implements RepositoryPlugin { - public static final String FAKE_VERSIONS_TYPE = "fakeversionsrepo"; - - @Override - public Map getRepositories( - Environment env, - NamedXContentRegistry namedXContentRegistry, - ClusterService clusterService, - BigArrays bigArrays, - RecoverySettings recoverySettings - ) { - return Map.of( - FAKE_VERSIONS_TYPE, - metadata -> new FakeVersionsRepo(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) - ); - } - - // fakes an old index version format to activate license checks - private static class FakeVersionsRepo extends FsRepository { - FakeVersionsRepo( - RepositoryMetadata metadata, - Environment env, - NamedXContentRegistry namedXContentRegistry, - ClusterService clusterService, - BigArrays bigArrays, - RecoverySettings recoverySettings - ) { - super(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); - } - - @Override - public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, SnapshotId snapshotId, IndexId index) - throws IOException { - final IndexMetadata original = super.getSnapshotIndexMetaData(repositoryData, snapshotId, index); - return IndexMetadata.builder(original) - .settings( - Settings.builder() - .put(original.getSettings()) - .put( - IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), - metadata.settings() - .getAsVersion("version", randomBoolean() ? Version.fromString("5.0.0") : Version.fromString("6.0.0")) - ) - ) - .build(); - } - } - } - - protected static final String repoName = "test-repo"; - protected static final String indexName = "test-index"; - protected static final String snapshotName = "test-snapshot"; - - @Before - public void createAndRestoreArchive() throws Exception { - createRepository(repoName, TestRepositoryPlugin.FAKE_VERSIONS_TYPE); - createIndex(indexName); - createFullSnapshot(repoName, snapshotName); - - assertAcked(client().admin().indices().prepareDelete(indexName)); - - PostStartTrialRequest request = new PostStartTrialRequest().setType(License.LicenseType.TRIAL.getTypeName()).acknowledge(true); - client().execute(PostStartTrialAction.INSTANCE, request).get(); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java deleted file mode 100644 index a9976b6f8c9f1..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.elasticsearch.ElasticsearchSecurityException; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.license.DeleteLicenseAction; -import org.elasticsearch.license.License; -import org.elasticsearch.license.LicensesMetadata; -import org.elasticsearch.license.PostStartBasicAction; -import org.elasticsearch.license.PostStartBasicRequest; -import org.elasticsearch.license.PostStartTrialAction; -import org.elasticsearch.license.PostStartTrialRequest; -import org.elasticsearch.license.PostStartTrialResponse; -import org.elasticsearch.protocol.xpack.XPackUsageRequest; -import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest; -import org.elasticsearch.snapshots.SnapshotRestoreException; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; -import org.elasticsearch.xpack.core.archive.ArchiveFeatureSetUsage; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.oneOf; - -public class ArchiveLicenseIntegTests extends AbstractArchiveTestCase { - - public void testFeatureUsage() throws Exception { - XPackUsageFeatureResponse usage = client().execute(XPackUsageFeatureAction.ARCHIVE, new XPackUsageRequest()).get(); - assertThat(usage.getUsage(), instanceOf(ArchiveFeatureSetUsage.class)); - ArchiveFeatureSetUsage archiveUsage = (ArchiveFeatureSetUsage) usage.getUsage(); - assertEquals(0, archiveUsage.getNumberOfArchiveIndices()); - - final RestoreSnapshotRequest req = new RestoreSnapshotRequest(repoName, snapshotName).indices(indexName).waitForCompletion(true); - - final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().restoreSnapshot(req).get(); - assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); - ensureGreen(indexName); - - usage = client().execute(XPackUsageFeatureAction.ARCHIVE, new XPackUsageRequest()).get(); - assertThat(usage.getUsage(), instanceOf(ArchiveFeatureSetUsage.class)); - archiveUsage = (ArchiveFeatureSetUsage) usage.getUsage(); - assertEquals(1, archiveUsage.getNumberOfArchiveIndices()); - } - - public void testFailRestoreOnInvalidLicense() throws Exception { - assertAcked(client().execute(DeleteLicenseAction.INSTANCE, new DeleteLicenseRequest()).get()); - assertAcked(client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest()).get()); - - ensureClusterSizeConsistency(); - ensureClusterStateConsistency(); - - final RestoreSnapshotRequest req = new RestoreSnapshotRequest(repoName, snapshotName).indices(indexName).waitForCompletion(true); - ElasticsearchSecurityException e = expectThrows( - ElasticsearchSecurityException.class, - () -> client().admin().cluster().restoreSnapshot(req).actionGet() - ); - assertThat(e.getMessage(), containsString("current license is non-compliant for [archive]")); - } - - public void testFailRestoreOnTooOldVersion() { - createRepository( - repoName, - TestRepositoryPlugin.FAKE_VERSIONS_TYPE, - Settings.builder().put(getRepositoryOnMaster(repoName).getMetadata().settings()).put("version", Version.fromString("2.0.0").id) - ); - final RestoreSnapshotRequest req = new RestoreSnapshotRequest(repoName, snapshotName).indices(indexName).waitForCompletion(true); - SnapshotRestoreException e = expectThrows( - SnapshotRestoreException.class, - () -> client().admin().cluster().restoreSnapshot(req).actionGet() - ); - assertThat( - e.getMessage(), - containsString( - "the snapshot was created with Elasticsearch version [2.0.0] " + "which isn't supported by the archive functionality" - ) - ); - } - - // checks that shards are failed if license becomes invalid after successful restore - public void testShardAllocationOnInvalidLicense() throws Exception { - final RestoreSnapshotRequest req = new RestoreSnapshotRequest(repoName, snapshotName).indices(indexName).waitForCompletion(true); - - final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().restoreSnapshot(req).get(); - assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); - ensureGreen(indexName); - - assertAcked(client().execute(DeleteLicenseAction.INSTANCE, new DeleteLicenseRequest()).get()); - assertAcked(client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest()).get()); - - ensureClusterSizeConsistency(); - ensureClusterStateConsistency(); - - // check that shards have been failed as part of invalid license - assertBusy( - () -> assertEquals( - ClusterHealthStatus.RED, - client().admin().cluster().prepareHealth(indexName).get().getIndices().get(indexName).getStatus() - ) - ); - - waitNoPendingTasksOnAll(); - ensureClusterStateConsistency(); - - // add a valid license again - // This is a bit of a hack in tests, as we can't readd a trial license - // We force this by clearing the existing basic license first - updateClusterState( - currentState -> ClusterState.builder(currentState) - .metadata(Metadata.builder(currentState.metadata()).removeCustom(LicensesMetadata.TYPE).build()) - .build() - ); - - waitNoPendingTasksOnAll(); - ensureClusterStateConsistency(); - - PostStartTrialRequest request = new PostStartTrialRequest().setType(License.LicenseType.TRIAL.getTypeName()).acknowledge(true); - final PostStartTrialResponse response = client().execute(PostStartTrialAction.INSTANCE, request).get(); - assertThat( - response.getStatus(), - oneOf( - PostStartTrialResponse.Status.UPGRADED_TO_TRIAL, - // The LicenceService automatically generates a license of {@link LicenceService#SELF_GENERATED_LICENSE_TYPE} type - // if there is no license found in the cluster state (see {@link LicenceService#registerOrUpdateSelfGeneratedLicense). - // Since this test explicitly removes the LicensesMetadata from cluster state it is possible that the self generated - // license is created before the PostStartTrialRequest is acked. - PostStartTrialResponse.Status.TRIAL_ALREADY_ACTIVATED - ) - ); - // check if cluster goes green again after valid license has been put in place - ensureGreen(indexName); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveSettingValidationIntegTests.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveSettingValidationIntegTests.java deleted file mode 100644 index 3a558a7c61668..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveSettingValidationIntegTests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; - -import java.util.concurrent.ExecutionException; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class ArchiveSettingValidationIntegTests extends AbstractArchiveTestCase { - public void testCannotRemoveWriteBlock() throws ExecutionException, InterruptedException { - final RestoreSnapshotRequest req = new RestoreSnapshotRequest(repoName, snapshotName).indices(indexName).waitForCompletion(true); - - final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().restoreSnapshot(req).get(); - assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); - ensureGreen(indexName); - - final IllegalArgumentException iae = expectThrows( - IllegalArgumentException.class, - () -> client().admin() - .indices() - .prepareUpdateSettings(indexName) - .setSettings(Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), false)) - .get() - ); - assertThat( - iae.getMessage(), - containsString("illegal value can't update [" + IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey() + "] from [true] to [false]") - ); - assertNotNull(iae.getCause()); - assertThat(iae.getCause().getMessage(), containsString("Cannot remove write block from archive index")); - - client().admin() - .indices() - .prepareUpdateSettings(indexName) - .setSettings(Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), true)) - .get(); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/LocalStateOldLuceneVersions.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/LocalStateOldLuceneVersions.java deleted file mode 100644 index e4a6110be7693..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/LocalStateOldLuceneVersions.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; - -import java.nio.file.Path; - -public class LocalStateOldLuceneVersions extends LocalStateCompositeXPackPlugin { - - private final OldLuceneVersions plugin; - - public LocalStateOldLuceneVersions(final Settings settings, final Path configPath) { - super(settings, configPath); - this.plugin = new OldLuceneVersions() { - - @Override - protected XPackLicenseState getLicenseState() { - return LocalStateOldLuceneVersions.this.getLicenseState(); - } - - }; - plugins.add(plugin); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveAllocationDecider.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveAllocationDecider.java deleted file mode 100644 index dde63f0aae0e3..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveAllocationDecider.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; -import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; -import org.elasticsearch.cluster.routing.allocation.decider.Decision; - -import java.util.function.BooleanSupplier; - -public class ArchiveAllocationDecider extends AllocationDecider { - - static final String NAME = "archive"; - - private final BooleanSupplier hasValidLicenseSupplier; - - public ArchiveAllocationDecider(BooleanSupplier hasValidLicenseSupplier) { - this.hasValidLicenseSupplier = hasValidLicenseSupplier; - } - - @Override - public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return allowAllocation(allocation.metadata().getIndexSafe(shardRouting.index()), allocation); - } - - @Override - public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) { - return allowAllocation(allocation.metadata().getIndexSafe(shardRouting.index()), allocation); - } - - @Override - public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, RoutingAllocation allocation) { - return allowAllocation(indexMetadata, allocation); - } - - @Override - public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return allowAllocation(allocation.metadata().getIndexSafe(shardRouting.index()), allocation); - } - - private Decision allowAllocation(IndexMetadata indexMetadata, RoutingAllocation allocation) { - if (indexMetadata.getCreationVersion().isLegacyIndexVersion()) { - if (hasValidLicenseSupplier.getAsBoolean()) { - return allocation.decision(Decision.YES, NAME, "valid license for archive functionality"); - } else { - return allocation.decision(Decision.NO, NAME, "invalid license for archive functionality"); - } - } else { - return allocation.decision(Decision.YES, NAME, "decider only applicable for indices backed by archive functionality"); - } - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveInfoTransportAction.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveInfoTransportAction.java deleted file mode 100644 index 702559a4810d8..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveInfoTransportAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.XPackField; -import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; -import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; - -import static org.elasticsearch.xpack.lucene.bwc.OldLuceneVersions.ARCHIVE_FEATURE; - -public class ArchiveInfoTransportAction extends XPackInfoFeatureTransportAction { - - private final XPackLicenseState licenseState; - - @Inject - public ArchiveInfoTransportAction(TransportService transportService, ActionFilters actionFilters, XPackLicenseState licenseState) { - super(XPackInfoFeatureAction.ARCHIVE.name(), transportService, actionFilters); - this.licenseState = licenseState; - } - - @Override - public String name() { - return XPackField.ARCHIVE; - } - - @Override - public boolean available() { - return ARCHIVE_FEATURE.checkWithoutTracking(licenseState); - } - - @Override - public boolean enabled() { - return true; - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTracker.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTracker.java deleted file mode 100644 index fcf47f7c580e6..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTracker.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.license.XPackLicenseState; - -import java.util.function.Supplier; - -import static org.elasticsearch.xpack.lucene.bwc.OldLuceneVersions.ARCHIVE_FEATURE; - -final class ArchiveUsageTracker implements Runnable { - - private final XPackLicenseState licenseState; - private final Supplier clusterStateSupplier; - - ArchiveUsageTracker(XPackLicenseState licenseState, Supplier clusterStateSupplier) { - this.clusterStateSupplier = clusterStateSupplier; - this.licenseState = licenseState; - } - - @Override - public void run() { - if (hasArchiveIndices(clusterStateSupplier.get())) { - ARCHIVE_FEATURE.check(licenseState); - } - } - - private static boolean hasArchiveIndices(ClusterState state) { - for (IndexMetadata indexMetadata : state.metadata()) { - if (indexMetadata.getCreationVersion().isLegacyIndexVersion()) { - return true; - } - } - return false; - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTransportAction.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTransportAction.java deleted file mode 100644 index e4e4c67adbc04..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTransportAction.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.protocol.xpack.XPackUsageRequest; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; -import org.elasticsearch.xpack.core.archive.ArchiveFeatureSetUsage; - -import static org.elasticsearch.xpack.lucene.bwc.OldLuceneVersions.ARCHIVE_FEATURE; - -public class ArchiveUsageTransportAction extends XPackUsageFeatureTransportAction { - - private final XPackLicenseState licenseState; - - @Inject - public ArchiveUsageTransportAction( - TransportService transportService, - ClusterService clusterService, - ThreadPool threadPool, - ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, - XPackLicenseState licenseState - ) { - super( - XPackUsageFeatureAction.ARCHIVE.name(), - transportService, - clusterService, - threadPool, - actionFilters, - indexNameExpressionResolver - ); - this.licenseState = licenseState; - } - - @Override - protected void masterOperation( - Task task, - XPackUsageRequest request, - ClusterState state, - ActionListener listener - ) { - int numArchiveIndices = 0; - for (IndexMetadata indexMetadata : state.metadata()) { - if (indexMetadata.getCreationVersion().isLegacyIndexVersion()) { - numArchiveIndices++; - } - } - listener.onResponse( - new XPackUsageFeatureResponse(new ArchiveFeatureSetUsage(ARCHIVE_FEATURE.checkWithoutTracking(licenseState), numArchiveIndices)) - ); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/FailShardsOnInvalidLicenseClusterListener.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/FailShardsOnInvalidLicenseClusterListener.java deleted file mode 100644 index 2cf7160518d74..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/FailShardsOnInvalidLicenseClusterListener.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.lucene.store.AlreadyClosedException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.routing.RerouteService; -import org.elasticsearch.common.Priority; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.shard.IndexEventListener; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.license.LicenseStateListener; -import org.elasticsearch.license.XPackLicenseState; - -import java.util.HashSet; -import java.util.Set; - -import static org.elasticsearch.xpack.lucene.bwc.OldLuceneVersions.ARCHIVE_FEATURE; - -public class FailShardsOnInvalidLicenseClusterListener implements LicenseStateListener, IndexEventListener { - - private static final Logger logger = LogManager.getLogger(FailShardsOnInvalidLicenseClusterListener.class); - - private final XPackLicenseState xPackLicenseState; - - private final RerouteService rerouteService; - - final Set shardsToFail = new HashSet<>(); - - private boolean allowed; - - public FailShardsOnInvalidLicenseClusterListener(XPackLicenseState xPackLicenseState, RerouteService rerouteService) { - this.xPackLicenseState = xPackLicenseState; - this.rerouteService = rerouteService; - this.allowed = ARCHIVE_FEATURE.checkWithoutTracking(xPackLicenseState); - xPackLicenseState.addListener(this); - } - - @Override - public synchronized void afterIndexShardStarted(IndexShard indexShard) { - shardsToFail.add(indexShard); - failActiveShardsIfNecessary(); - } - - @Override - public synchronized void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { - if (indexShard != null) { - shardsToFail.remove(indexShard); - } - } - - @Override - public synchronized void licenseStateChanged() { - final boolean allowed = ARCHIVE_FEATURE.checkWithoutTracking(xPackLicenseState); - if (allowed && this.allowed == false) { - rerouteService.reroute("reroute after license activation", Priority.NORMAL, new ActionListener() { - @Override - public void onResponse(ClusterState clusterState) { - logger.trace("successful reroute after license activation"); - } - - @Override - public void onFailure(Exception e) { - logger.debug("unsuccessful reroute after license activation"); - } - }); - } - this.allowed = allowed; - failActiveShardsIfNecessary(); - } - - private void failActiveShardsIfNecessary() { - assert Thread.holdsLock(this); - if (allowed == false) { - for (IndexShard indexShard : shardsToFail) { - try { - indexShard.failShard("invalid license", null); - } catch (AlreadyClosedException ignored) { - // ignore - } catch (Exception e) { - logger.warn(new ParameterizedMessage("Could not close shard {} due to invalid license", indexShard.shardId()), e); - } - } - shardsToFail.clear(); - } - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java deleted file mode 100644 index 86b4b3eb1ef6a..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.apache.lucene.index.SegmentCommitInfo; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.util.SetOnce; -import org.elasticsearch.Version; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.index.IndexModule; -import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.seqno.SequenceNumbers; -import org.elasticsearch.index.shard.IndexEventListener; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.license.License; -import org.elasticsearch.license.LicenseUtils; -import org.elasticsearch.license.LicensedFeature; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.plugins.ActionPlugin; -import org.elasticsearch.plugins.ClusterPlugin; -import org.elasticsearch.plugins.IndexStorePlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.snapshots.Snapshot; -import org.elasticsearch.snapshots.SnapshotRestoreException; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; -import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.BiConsumer; -import java.util.function.Supplier; - -public class OldLuceneVersions extends Plugin implements IndexStorePlugin, ClusterPlugin, RepositoryPlugin, ActionPlugin { - - public static final LicensedFeature.Momentary ARCHIVE_FEATURE = LicensedFeature.momentary( - null, - "archive", - License.OperationMode.ENTERPRISE - ); - - private static Version MINIMUM_ARCHIVE_VERSION = Version.fromString("5.0.0"); - - private final SetOnce failShardsListener = new SetOnce<>(); - - @Override - public Collection createComponents( - final Client client, - final ClusterService clusterService, - final ThreadPool threadPool, - final ResourceWatcherService resourceWatcherService, - final ScriptService scriptService, - final NamedXContentRegistry xContentRegistry, - final Environment environment, - final NodeEnvironment nodeEnvironment, - final NamedWriteableRegistry registry, - final IndexNameExpressionResolver resolver, - final Supplier repositoriesServiceSupplier - ) { - this.failShardsListener.set(new FailShardsOnInvalidLicenseClusterListener(getLicenseState(), clusterService.getRerouteService())); - if (DiscoveryNode.isMasterNode(environment.settings())) { - // We periodically look through the indices and identify if there are any archive indices, - // then marking the feature as used. We do this on each master node so that if one master fails, the - // continue reporting usage state. - var usageTracker = new ArchiveUsageTracker(getLicenseState(), clusterService::state); - threadPool.scheduleWithFixedDelay(usageTracker, TimeValue.timeValueMinutes(15), ThreadPool.Names.GENERIC); - } - return List.of(); - } - - @Override - public List> getActions() { - return List.of( - new ActionPlugin.ActionHandler<>(XPackUsageFeatureAction.ARCHIVE, ArchiveUsageTransportAction.class), - new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.ARCHIVE, ArchiveInfoTransportAction.class) - ); - } - - // overridable by tests - protected XPackLicenseState getLicenseState() { - return XPackPlugin.getSharedLicenseState(); - } - - @Override - public Collection createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) { - return List.of(new ArchiveAllocationDecider(() -> ARCHIVE_FEATURE.checkWithoutTracking(getLicenseState()))); - } - - @Override - public void onIndexModule(IndexModule indexModule) { - if (indexModule.indexSettings().getIndexVersionCreated().isLegacyIndexVersion()) { - indexModule.addIndexEventListener(new IndexEventListener() { - @Override - public void afterFilesRestoredFromRepository(IndexShard indexShard) { - convertToNewFormat(indexShard); - } - }); - - indexModule.addIndexEventListener(failShardsListener.get()); - - indexModule.addSettingsUpdateConsumer(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING, s -> {}, write -> { - if (write == false) { - throw new IllegalArgumentException("Cannot remove write block from archive index"); - } - }); - } - } - - @Override - public BiConsumer addPreRestoreVersionCheck() { - return (snapshot, version) -> { - if (version.isLegacyIndexVersion()) { - if (ARCHIVE_FEATURE.checkWithoutTracking(getLicenseState()) == false) { - throw LicenseUtils.newComplianceException("archive"); - } - if (version.before(MINIMUM_ARCHIVE_VERSION)) { - throw new SnapshotRestoreException( - snapshot, - "the snapshot was created with Elasticsearch version [" - + version - + "] which isn't supported by the archive functionality" - ); - } - } - }; - } - - /** - * The trick used to allow newer Lucene versions to read older Lucene indices is to convert the old directory to a directory that new - * Lucene versions happily operate on. The way newer Lucene versions happily comply with reading older data is to put in place a - * segments file that the newer Lucene version can open, using codecs that allow reading everything from the old files, making it - * available under the newer interfaces. The way this works is to read in the old segments file using a special class - * {@link OldSegmentInfos} that supports reading older Lucene {@link SegmentInfos}, and then write out an updated segments file that - * newer Lucene versions can understand. - */ - private static void convertToNewFormat(IndexShard indexShard) { - indexShard.store().incRef(); - try { - final OldSegmentInfos oldSegmentInfos = OldSegmentInfos.readLatestCommit(indexShard.store().directory(), 6); - final SegmentInfos segmentInfos = convertToNewerLuceneVersion(oldSegmentInfos); - // write upgraded segments file - segmentInfos.commit(indexShard.store().directory()); - - // what we have written can be read using standard path - assert SegmentInfos.readLatestCommit(indexShard.store().directory()) != null; - - // clean older segments file - Lucene.pruneUnreferencedFiles(segmentInfos.getSegmentsFileName(), indexShard.store().directory()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } finally { - indexShard.store().decRef(); - } - } - - private static SegmentInfos convertToNewerLuceneVersion(OldSegmentInfos oldSegmentInfos) { - final SegmentInfos segmentInfos = new SegmentInfos(org.apache.lucene.util.Version.LATEST.major); - segmentInfos.version = oldSegmentInfos.version; - segmentInfos.counter = oldSegmentInfos.counter; - segmentInfos.setNextWriteGeneration(oldSegmentInfos.getGeneration() + 1); - final Map map = new HashMap<>(oldSegmentInfos.getUserData()); - if (map.containsKey(Engine.HISTORY_UUID_KEY) == false) { - map.put(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID()); - } - if (map.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) == false) { - map.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(SequenceNumbers.NO_OPS_PERFORMED)); - } - if (map.containsKey(SequenceNumbers.MAX_SEQ_NO) == false) { - map.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(SequenceNumbers.NO_OPS_PERFORMED)); - } - if (map.containsKey(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID) == false) { - map.put(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, "-1"); - } - segmentInfos.setUserData(map, false); - for (SegmentCommitInfo infoPerCommit : oldSegmentInfos.asList()) { - final SegmentInfo newInfo = BWCCodec.wrap(infoPerCommit.info); - final SegmentCommitInfo commitInfo = new SegmentCommitInfo( - newInfo, - infoPerCommit.getDelCount(), - infoPerCommit.getSoftDelCount(), - infoPerCommit.getDelGen(), - infoPerCommit.getFieldInfosGen(), - infoPerCommit.getDocValuesGen(), - infoPerCommit.getId() - ); - commitInfo.setDocValuesUpdatesFiles(infoPerCommit.getDocValuesUpdatesFiles()); - commitInfo.setFieldInfosFiles(infoPerCommit.getFieldInfosFiles()); - segmentInfos.add(commitInfo); - } - return segmentInfos; - } - - @Override - public Map getDirectoryFactories() { - return Map.of(); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java deleted file mode 100644 index e5de349203b3d..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java +++ /dev/null @@ -1,737 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ - -package org.elasticsearch.xpack.lucene.bwc; - -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.IndexNotFoundException; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.SegmentCommitInfo; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.StringHelper; -import org.apache.lucene.util.Version; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.core.SuppressForbidden; - -import java.io.EOFException; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.PrintStream; -import java.nio.file.NoSuchFileException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Clone of Lucene's SegmentInfos class that allows parsing older formats - */ -@SuppressWarnings("CheckStyle") -@SuppressForbidden(reason = "Lucene class") -public class OldSegmentInfos implements Cloneable, Iterable { - - /** - * Adds the {@link Version} that committed this segments_N file, as well as the {@link Version} - * of the oldest segment, since 5.3+ - */ - public static final int VERSION_53 = 6; - /** - * The version that added information about the Lucene version at the time when the index has been - * created. - */ - public static final int VERSION_70 = 7; - /** The version that updated segment name counter to be long instead of int. */ - public static final int VERSION_72 = 8; - /** The version that recorded softDelCount */ - public static final int VERSION_74 = 9; - /** The version that recorded SegmentCommitInfo IDs */ - public static final int VERSION_86 = 10; - - static final int VERSION_CURRENT = VERSION_86; - - /** Name of the generation reference file name */ - private static final String OLD_SEGMENTS_GEN = "segments.gen"; - - /** Used to name new segments. */ - public long counter; - - /** Counts how often the index has been changed. */ - public long version; - - private long generation; // generation of the "segments_N" for the next commit - private long lastGeneration; // generation of the "segments_N" file we last successfully read - // or wrote; this is normally the same as generation except if - // there was an IOException that had interrupted a commit - - /** Opaque Map<String, String> that user can specify during IndexWriter.commit */ - public Map userData = Collections.emptyMap(); - - private List segments = new ArrayList<>(); - - /** - * If non-null, information about loading segments_N files will be printed here. - */ - private static PrintStream infoStream = null; - - /** Id for this commit; only written starting with Lucene 5.0 */ - private byte[] id; - - /** Which Lucene version wrote this commit. */ - private Version luceneVersion; - - /** Version of the oldest segment in the index, or null if there are no segments. */ - private Version minSegmentLuceneVersion; - - /** The Lucene version major that was used to create the index. */ - private final int indexCreatedVersionMajor; - - /** - * Sole constructor. - * - * @param indexCreatedVersionMajor the Lucene version major at index creation time, or 6 if the - * index was created before 7.0 - */ - public OldSegmentInfos(int indexCreatedVersionMajor) { - if (indexCreatedVersionMajor > Version.LATEST.major) { - throw new IllegalArgumentException("indexCreatedVersionMajor is in the future: " + indexCreatedVersionMajor); - } - if (indexCreatedVersionMajor < 6) { - throw new IllegalArgumentException("indexCreatedVersionMajor must be >= 6, got: " + indexCreatedVersionMajor); - } - this.indexCreatedVersionMajor = indexCreatedVersionMajor; - } - - /** Returns {@link SegmentCommitInfo} at the provided index. */ - public SegmentCommitInfo info(int i) { - return segments.get(i); - } - - /** - * Get the generation of the most recent commit to the list of index files (N in the segments_N - * file). - * - * @param files -- array of file names to check - */ - public static long getLastCommitGeneration(String[] files) { - long max = -1; - for (String file : files) { - if (file.startsWith(IndexFileNames.SEGMENTS) && - // skipping this file here helps deliver the right exception when opening an old index - file.startsWith(OLD_SEGMENTS_GEN) == false) { - long gen = generationFromSegmentsFileName(file); - if (gen > max) { - max = gen; - } - } - } - return max; - } - - /** Get the segments_N filename in use by this segment infos. */ - public String getSegmentsFileName() { - return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", lastGeneration); - } - - /** Parse the generation off the segments file name and return it. */ - public static long generationFromSegmentsFileName(String fileName) { - if (fileName.equals(OLD_SEGMENTS_GEN)) { - throw new IllegalArgumentException("\"" + OLD_SEGMENTS_GEN + "\" is not a valid segment file name since 4.0"); - } else if (fileName.equals(IndexFileNames.SEGMENTS)) { - return 0; - } else if (fileName.startsWith(IndexFileNames.SEGMENTS)) { - return Long.parseLong(fileName.substring(1 + IndexFileNames.SEGMENTS.length()), Character.MAX_RADIX); - } else { - throw new IllegalArgumentException("fileName \"" + fileName + "\" is not a segments file"); - } - } - - /** return generation of the next pending_segments_N that will be written */ - private long getNextPendingGeneration() { - if (generation == -1) { - return 1; - } else { - return generation + 1; - } - } - - /** Since Lucene 5.0, every commit (segments_N) writes a unique id. This will return that id */ - public byte[] getId() { - return id.clone(); - } - - static final OldSegmentInfos readCommit(Directory directory, String segmentFileName, int minSupportedMajorVersion) throws IOException { - - long generation = generationFromSegmentsFileName(segmentFileName); - // System.out.println(Thread.currentThread() + ": SegmentInfos.readCommit " + segmentFileName); - try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ)) { - try { - return readCommit(directory, input, generation, minSupportedMajorVersion); - } catch (EOFException | NoSuchFileException | FileNotFoundException e) { - throw new CorruptIndexException("Unexpected file read error while reading index.", input, e); - } - } - } - - /** Read the commit from the provided {@link ChecksumIndexInput}. */ - static final OldSegmentInfos readCommit(Directory directory, ChecksumIndexInput input, long generation, int minSupportedMajorVersion) - throws IOException { - Throwable priorE = null; - int format = -1; - try { - // NOTE: as long as we want to throw indexformattooold (vs corruptindexexception), we need - // to read the magic ourselves. - int magic = CodecUtil.readBEInt(input); - if (magic != CodecUtil.CODEC_MAGIC) { - throw new IndexFormatTooOldException(input, magic, CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC); - } - format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_53, VERSION_CURRENT); - byte[] id = new byte[StringHelper.ID_LENGTH]; - input.readBytes(id, 0, id.length); - CodecUtil.checkIndexHeaderSuffix(input, Long.toString(generation, Character.MAX_RADIX)); - - Version luceneVersion = Version.fromBits(input.readVInt(), input.readVInt(), input.readVInt()); - int indexCreatedVersion = 6; - if (format >= VERSION_70) { - indexCreatedVersion = input.readVInt(); - } - if (luceneVersion.major < indexCreatedVersion) { - throw new CorruptIndexException( - "Creation version [" - + indexCreatedVersion - + ".x] can't be greater than the version that wrote the segment infos: [" - + luceneVersion - + "]", - input - ); - } - - if (indexCreatedVersion < minSupportedMajorVersion) { - throw new IndexFormatTooOldException( - input, - "This index was initially created with Lucene " - + indexCreatedVersion - + ".x while the current version is " - + Version.LATEST - + " and Lucene only supports reading" - + (minSupportedMajorVersion == Version.MIN_SUPPORTED_MAJOR - ? " the current and previous major versions" - : " from version " + minSupportedMajorVersion + " upwards") - ); - } - - OldSegmentInfos infos = new OldSegmentInfos(indexCreatedVersion); - infos.id = id; - infos.generation = generation; - infos.lastGeneration = generation; - infos.luceneVersion = luceneVersion; - parseSegmentInfos(directory, input, infos, format); - return infos; - - } catch (Throwable t) { - priorE = t; - } finally { - if (format >= VERSION_53) { // oldest supported version - CodecUtil.checkFooter(input, priorE); - } else { - throw IOUtils.rethrowAlways(priorE); - } - } - throw new Error("Unreachable code"); - } - - private static void parseSegmentInfos(Directory directory, DataInput input, OldSegmentInfos infos, int format) throws IOException { - infos.version = CodecUtil.readBELong(input); - // System.out.println("READ sis version=" + infos.version); - if (format > VERSION_70) { - infos.counter = input.readVLong(); - } else { - infos.counter = CodecUtil.readBEInt(input); - } - int numSegments = CodecUtil.readBEInt(input); - if (numSegments < 0) { - throw new CorruptIndexException("invalid segment count: " + numSegments, input); - } - - if (numSegments > 0) { - infos.minSegmentLuceneVersion = Version.fromBits(input.readVInt(), input.readVInt(), input.readVInt()); - } else { - // else leave as null: no segments - } - - long totalDocs = 0; - for (int seg = 0; seg < numSegments; seg++) { - String segName = input.readString(); - if (format < VERSION_70) { - byte hasID = input.readByte(); - if (hasID == 0) { - throw new IndexFormatTooOldException(input, "Segment is from Lucene 4.x"); - } else if (hasID != 1) { - throw new CorruptIndexException("invalid hasID byte, got: " + hasID, input); - } - } - byte[] segmentID = new byte[StringHelper.ID_LENGTH]; - input.readBytes(segmentID, 0, segmentID.length); - Codec codec = readCodec(input); - SegmentInfo info = codec.segmentInfoFormat().read(directory, segName, segmentID, IOContext.READ); - info.setCodec(codec); - totalDocs += info.maxDoc(); - long delGen = CodecUtil.readBELong(input); - int delCount = CodecUtil.readBEInt(input); - if (delCount < 0 || delCount > info.maxDoc()) { - throw new CorruptIndexException("invalid deletion count: " + delCount + " vs maxDoc=" + info.maxDoc(), input); - } - long fieldInfosGen = CodecUtil.readBELong(input); - long dvGen = CodecUtil.readBELong(input); - int softDelCount = format > VERSION_72 ? CodecUtil.readBEInt(input) : 0; - if (softDelCount < 0 || softDelCount > info.maxDoc()) { - throw new CorruptIndexException("invalid deletion count: " + softDelCount + " vs maxDoc=" + info.maxDoc(), input); - } - if (softDelCount + delCount > info.maxDoc()) { - throw new CorruptIndexException( - "invalid deletion count: " + (softDelCount + delCount) + " vs maxDoc=" + info.maxDoc(), - input - ); - } - final byte[] sciId; - if (format > VERSION_74) { - byte marker = input.readByte(); - switch (marker) { - case 1 -> { - sciId = new byte[StringHelper.ID_LENGTH]; - input.readBytes(sciId, 0, sciId.length); - } - case 0 -> sciId = null; - default -> throw new CorruptIndexException("invalid SegmentCommitInfo ID marker: " + marker, input); - } - } else { - sciId = null; - } - SegmentCommitInfo siPerCommit = new SegmentCommitInfo(info, delCount, softDelCount, delGen, fieldInfosGen, dvGen, sciId); - siPerCommit.setFieldInfosFiles(input.readSetOfStrings()); - final Map> dvUpdateFiles; - final int numDVFields = CodecUtil.readBEInt(input); - if (numDVFields == 0) { - dvUpdateFiles = Collections.emptyMap(); - } else { - Map> map = Maps.newMapWithExpectedSize(numDVFields); - for (int i = 0; i < numDVFields; i++) { - map.put(CodecUtil.readBEInt(input), input.readSetOfStrings()); - } - dvUpdateFiles = Collections.unmodifiableMap(map); - } - siPerCommit.setDocValuesUpdatesFiles(dvUpdateFiles); - infos.add(siPerCommit); - - Version segmentVersion = info.getVersion(); - - if (segmentVersion.onOrAfter(infos.minSegmentLuceneVersion) == false) { - throw new CorruptIndexException( - "segments file recorded minSegmentLuceneVersion=" - + infos.minSegmentLuceneVersion - + " but segment=" - + info - + " has older version=" - + segmentVersion, - input - ); - } - - if (infos.indexCreatedVersionMajor >= 7 && segmentVersion.major < infos.indexCreatedVersionMajor) { - throw new CorruptIndexException( - "segments file recorded indexCreatedVersionMajor=" - + infos.indexCreatedVersionMajor - + " but segment=" - + info - + " has older version=" - + segmentVersion, - input - ); - } - - if (infos.indexCreatedVersionMajor >= 7 && info.getMinVersion() == null) { - throw new CorruptIndexException( - "segments infos must record minVersion with indexCreatedVersionMajor=" + infos.indexCreatedVersionMajor, - input - ); - } - } - - infos.userData = input.readMapOfStrings(); - } - - private static Codec readCodec(DataInput input) throws IOException { - final String name = input.readString(); - try { - return Codec.forName(name); - } catch (IllegalArgumentException e) { - // maybe it's an old default codec that moved - if (name.startsWith("Lucene")) { - throw new IllegalArgumentException( - "Could not load codec '" + name + "'. Did you forget to add lucene-backward-codecs.jar?", - e - ); - } - throw e; - } - } - - /** Find the latest commit ({@code segments_N file}) and load all {@link SegmentCommitInfo}s. */ - public static final OldSegmentInfos readLatestCommit(Directory directory) throws IOException { - return readLatestCommit(directory, Version.MIN_SUPPORTED_MAJOR); - } - - static final OldSegmentInfos readLatestCommit(Directory directory, int minSupportedMajorVersion) throws IOException { - return new OldSegmentInfos.FindSegmentsFile(directory) { - @Override - protected OldSegmentInfos doBody(String segmentFileName) throws IOException { - return readCommit(directory, segmentFileName, minSupportedMajorVersion); - } - }.run(); - } - - // Only true after prepareCommit has been called and - // before finishCommit is called - boolean pendingCommit; - - private void write(Directory directory) throws IOException { - - long nextGeneration = getNextPendingGeneration(); - String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", nextGeneration); - - // Always advance the generation on write: - generation = nextGeneration; - - IndexOutput segnOutput = null; - boolean success = false; - - try { - segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT); - write(segnOutput); - segnOutput.close(); - directory.sync(Collections.singleton(segmentFileName)); - success = true; - } finally { - if (success) { - pendingCommit = true; - } else { - // We hit an exception above; try to close the file - // but suppress any exception: - IOUtils.closeWhileHandlingException(segnOutput); - // Try not to leave a truncated segments_N file in - // the index: - IOUtils.deleteFilesIgnoringExceptions(directory, segmentFileName); - } - } - } - - /** Write ourselves to the provided {@link IndexOutput} */ - public void write(IndexOutput out) throws IOException { - CodecUtil.writeIndexHeader( - out, - "segments", - VERSION_CURRENT, - StringHelper.randomId(), - Long.toString(generation, Character.MAX_RADIX) - ); - out.writeVInt(Version.LATEST.major); - out.writeVInt(Version.LATEST.minor); - out.writeVInt(Version.LATEST.bugfix); - // System.out.println(Thread.currentThread().getName() + ": now write " + out.getName() + " with - // version=" + version); - - out.writeVInt(indexCreatedVersionMajor); - - CodecUtil.writeBELong(out, version); - out.writeVLong(counter); // write counter - CodecUtil.writeBEInt(out, size()); - - if (size() > 0) { - - Version minSegmentVersion = null; - - // We do a separate loop up front so we can write the minSegmentVersion before - // any SegmentInfo; this makes it cleaner to throw IndexFormatTooOldExc at read time: - for (SegmentCommitInfo siPerCommit : this) { - Version segmentVersion = siPerCommit.info.getVersion(); - if (minSegmentVersion == null || segmentVersion.onOrAfter(minSegmentVersion) == false) { - minSegmentVersion = segmentVersion; - } - } - - out.writeVInt(minSegmentVersion.major); - out.writeVInt(minSegmentVersion.minor); - out.writeVInt(minSegmentVersion.bugfix); - } - - // write infos - for (SegmentCommitInfo siPerCommit : this) { - SegmentInfo si = siPerCommit.info; - out.writeString(si.name); - byte[] segmentID = si.getId(); - if (segmentID.length != StringHelper.ID_LENGTH) { - throw new IllegalStateException( - "cannot write segment: invalid id segment=" + si.name + "id=" + StringHelper.idToString(segmentID) - ); - } - out.writeBytes(segmentID, segmentID.length); - out.writeString(si.getCodec().getName()); - - CodecUtil.writeBELong(out, siPerCommit.getDelGen()); - int delCount = siPerCommit.getDelCount(); - if (delCount < 0 || delCount > si.maxDoc()) { - throw new IllegalStateException( - "cannot write segment: invalid maxDoc segment=" + si.name + " maxDoc=" + si.maxDoc() + " delCount=" + delCount - ); - } - CodecUtil.writeBEInt(out, delCount); - CodecUtil.writeBELong(out, siPerCommit.getFieldInfosGen()); - CodecUtil.writeBELong(out, siPerCommit.getDocValuesGen()); - int softDelCount = siPerCommit.getSoftDelCount(); - if (softDelCount < 0 || softDelCount > si.maxDoc()) { - throw new IllegalStateException( - "cannot write segment: invalid maxDoc segment=" + si.name + " maxDoc=" + si.maxDoc() + " softDelCount=" + softDelCount - ); - } - CodecUtil.writeBEInt(out, softDelCount); - // we ensure that there is a valid ID for this SCI just in case - // this is manually upgraded outside of IW - byte[] sciId = siPerCommit.getId(); - if (sciId != null) { - out.writeByte((byte) 1); - assert sciId.length == StringHelper.ID_LENGTH : "invalid SegmentCommitInfo#id: " + Arrays.toString(sciId); - out.writeBytes(sciId, 0, sciId.length); - } else { - out.writeByte((byte) 0); - } - - out.writeSetOfStrings(siPerCommit.getFieldInfosFiles()); - final Map> dvUpdatesFiles = siPerCommit.getDocValuesUpdatesFiles(); - CodecUtil.writeBEInt(out, dvUpdatesFiles.size()); - for (Map.Entry> e : dvUpdatesFiles.entrySet()) { - CodecUtil.writeBEInt(out, e.getKey()); - out.writeSetOfStrings(e.getValue()); - } - } - out.writeMapOfStrings(userData); - CodecUtil.writeFooter(out); - } - - /** version number when this SegmentInfos was generated. */ - public long getVersion() { - return version; - } - - /** Returns current generation. */ - public long getGeneration() { - return generation; - } - - /** Returns last succesfully read or written generation. */ - public long getLastGeneration() { - return lastGeneration; - } - - /** - * Prints the given message to the infoStream. Note, this method does not check for null - * infoStream. It assumes this check has been performed by the caller, which is recommended to - * avoid the (usually) expensive message creation. - */ - private static void message(String message) { - infoStream.println("SIS [" + Thread.currentThread().getName() + "]: " + message); - } - - /** - * Utility class for executing code that needs to do something with the current segments file. - * This is necessary with lock-less commits because from the time you locate the current segments - * file name, until you actually open it, read its contents, or check modified time, etc., it - * could have been deleted due to a writer commit finishing. - */ - @SuppressWarnings("CheckStyle") - public abstract static class FindSegmentsFile { - - final Directory directory; - - /** Sole constructor. */ - protected FindSegmentsFile(Directory directory) { - this.directory = directory; - } - - /** Locate the most recent {@code segments} file and run {@link #doBody} on it. */ - public T run() throws IOException { - return run(null); - } - - /** Run {@link #doBody} on the provided commit. */ - public T run(IndexCommit commit) throws IOException { - if (commit != null) { - if (directory != commit.getDirectory()) throw new IOException( - "the specified commit does not match the specified Directory" - ); - return doBody(commit.getSegmentsFileName()); - } - - long lastGen = -1; - long gen = -1; - IOException exc = null; - - // Loop until we succeed in calling doBody() without - // hitting an IOException. An IOException most likely - // means an IW deleted our commit while opening - // the time it took us to load the now-old infos files - // (and segments files). It's also possible it's a - // true error (corrupt index). To distinguish these, - // on each retry we must see "forward progress" on - // which generation we are trying to load. If we - // don't, then the original error is real and we throw - // it. - - for (;;) { - lastGen = gen; - String[] files = directory.listAll(); - String[] files2 = directory.listAll(); - Arrays.sort(files); - Arrays.sort(files2); - if (Arrays.equals(files, files2) == false) { - // listAll() is weakly consistent, this means we hit "concurrent modification exception" - continue; - } - gen = getLastCommitGeneration(files); - - if (infoStream != null) { - message("directory listing gen=" + gen); - } - - if (gen == -1) { - throw new IndexNotFoundException("no segments* file found in " + directory + ": files: " + Arrays.toString(files)); - } else if (gen > lastGen) { - String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen); - - try { - T t = doBody(segmentFileName); - if (infoStream != null) { - message("success on " + segmentFileName); - } - return t; - } catch (IOException err) { - // Save the original root cause: - if (exc == null) { - exc = err; - } - - if (infoStream != null) { - message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: gen = " + gen); - } - } - } else { - throw exc; - } - } - } - - /** - * Subclass must implement this. The assumption is an IOException will be thrown if something - * goes wrong during the processing that could have been caused by a writer committing. - */ - protected abstract T doBody(String segmentFileName) throws IOException; - } - - /** - * Returns all file names referenced by SegmentInfo. The returned collection is recomputed on each - * invocation. - */ - public Collection files(boolean includeSegmentsFile) throws IOException { - HashSet files = new HashSet<>(); - if (includeSegmentsFile) { - final String segmentFileName = getSegmentsFileName(); - if (segmentFileName != null) { - files.add(segmentFileName); - } - } - final int size = size(); - for (int i = 0; i < size; i++) { - final SegmentCommitInfo info = info(i); - files.addAll(info.files()); - } - - return files; - } - - /** Returns readable description of this segment. */ - @Override - public String toString() { - StringBuilder buffer = new StringBuilder(); - buffer.append(getSegmentsFileName()).append(": "); - final int count = size(); - for (int i = 0; i < count; i++) { - if (i > 0) { - buffer.append(' '); - } - final SegmentCommitInfo info = info(i); - buffer.append(info.toString(0)); - } - return buffer.toString(); - } - - /** - * Return {@code userData} saved with this commit. - * - * @see IndexWriter#commit() - */ - public Map getUserData() { - return userData; - } - - /** Returns an unmodifiable {@link Iterator} of contained segments in order. */ - @Override - public Iterator iterator() { - return asList().iterator(); - } - - /** Returns all contained segments as an unmodifiable {@link List} view. */ - public List asList() { - return Collections.unmodifiableList(segments); - } - - /** Returns number of {@link SegmentCommitInfo}s. */ - public int size() { - return segments.size(); - } - - /** Appends the provided {@link SegmentCommitInfo}. */ - public void add(SegmentCommitInfo si) { - segments.add(si); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java deleted file mode 100644 index b350f6a62404f..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc.codecs; - -import org.apache.lucene.backward_codecs.lucene70.Lucene70Codec; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.codecs.FieldsProducer; -import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.NormsFormat; -import org.apache.lucene.codecs.NormsProducer; -import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.codecs.TermVectorsFormat; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.Terms; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -/** - * Base class for older BWC codecs - */ -public abstract class BWCCodec extends Codec { - - private final PostingsFormat postingsFormat = new EmptyPostingsFormat(); - - protected BWCCodec(String name) { - super(name); - } - - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; - } - - @Override - public NormsFormat normsFormat() { - throw new UnsupportedOperationException(); - } - - @Override - public TermVectorsFormat termVectorsFormat() { - throw new UnsupportedOperationException(); - } - - @Override - public PointsFormat pointsFormat() { - throw new UnsupportedOperationException(); - } - - @Override - public KnnVectorsFormat knnVectorsFormat() { - throw new UnsupportedOperationException(); - } - - /** - * In-memory postings format that shows no postings available. - * TODO: Remove once https://issues.apache.org/jira/browse/LUCENE-10291 is fixed. - */ - static class EmptyPostingsFormat extends PostingsFormat { - - protected EmptyPostingsFormat() { - super("EmptyPostingsFormat"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) { - return new FieldsConsumer() { - @Override - public void write(Fields fields, NormsProducer norms) { - throw new UnsupportedOperationException(); - } - - @Override - public void close() { - - } - }; - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) { - return new FieldsProducer() { - @Override - public void close() { - - } - - @Override - public void checkIntegrity() { - - } - - @Override - public Iterator iterator() { - return null; - } - - @Override - public Terms terms(String field) { - return null; - } - - @Override - public int size() { - return 0; - } - }; - } - } - - protected static SegmentInfoFormat wrap(SegmentInfoFormat wrapped) { - return new SegmentInfoFormat() { - @Override - public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException { - return wrap(wrapped.read(directory, segmentName, segmentID, context)); - } - - @Override - public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException { - wrapped.write(dir, info, ioContext); - } - }; - } - - protected static FieldInfosFormat wrap(FieldInfosFormat wrapped) { - return new FieldInfosFormat() { - @Override - public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) - throws IOException { - return filterFields(wrapped.read(directory, segmentInfo, segmentSuffix, iocontext)); - } - - @Override - public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) - throws IOException { - wrapped.write(directory, segmentInfo, segmentSuffix, infos, context); - } - }; - } - - // mark all fields as having no postings, no term vectors, no norms, no payloads, no points, and no vectors. - private static FieldInfos filterFields(FieldInfos fieldInfos) { - List fieldInfoCopy = new ArrayList<>(fieldInfos.size()); - for (FieldInfo fieldInfo : fieldInfos) { - fieldInfoCopy.add( - new FieldInfo( - fieldInfo.name, - fieldInfo.number, - false, - false, - false, - IndexOptions.NONE, - fieldInfo.getDocValuesType(), - fieldInfo.getDocValuesGen(), - fieldInfo.attributes(), - 0, - 0, - 0, - 0, - fieldInfo.getVectorSimilarityFunction(), - fieldInfo.isSoftDeletesField() - ) - ); - } - FieldInfos newFieldInfos = new FieldInfos(fieldInfoCopy.toArray(new FieldInfo[0])); - return newFieldInfos; - } - - public static SegmentInfo wrap(SegmentInfo segmentInfo) { - // special handling for Lucene70Codec (which is currently bundled with Lucene) - // Use BWCLucene70Codec instead as that one extends BWCCodec (similar to all other older codecs) - final Codec codec = segmentInfo.getCodec() instanceof Lucene70Codec ? new BWCLucene70Codec() : segmentInfo.getCodec(); - final SegmentInfo segmentInfo1 = new SegmentInfo( - segmentInfo.dir, - // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) - // TODO: perhaps store the original version information in attributes so that we can retrieve it later when needed? - org.apache.lucene.util.Version.LATEST, - org.apache.lucene.util.Version.LATEST, - segmentInfo.name, - segmentInfo.maxDoc(), - segmentInfo.getUseCompoundFile(), - codec, - segmentInfo.getDiagnostics(), - segmentInfo.getId(), - segmentInfo.getAttributes(), - segmentInfo.getIndexSort() - ); - segmentInfo1.setFiles(segmentInfo.files()); - return segmentInfo1; - } - -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValues.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValues.java deleted file mode 100644 index a7b1724fa24f0..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValues.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.util.BytesRef; - -/** - * A per-document byte[] - * - * @deprecated Use {@link BinaryDocValues} instead. - */ -@Deprecated -public abstract class LegacyBinaryDocValues { - - /** Sole constructor. (For invocation by subclass - * constructors, typically implicit.) */ - protected LegacyBinaryDocValues() {} - - /** Lookup the value for document. The returned {@link BytesRef} may be - * re-used across calls to {@link #get(int)} so make sure to - * {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want to keep it - * around. */ - public abstract BytesRef get(int docID); -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValuesWrapper.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValuesWrapper.java deleted file mode 100644 index 8dbacf889172e..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValuesWrapper.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; - -import java.io.IOException; - -/** - * Wraps a {@link LegacyBinaryDocValues} into a {@link BinaryDocValues}. - * - * @deprecated Implement {@link BinaryDocValues} directly. - */ -@Deprecated -public final class LegacyBinaryDocValuesWrapper extends BinaryDocValues { - private final Bits docsWithField; - private final LegacyBinaryDocValues values; - private final int maxDoc; - private int docID = -1; - - public LegacyBinaryDocValuesWrapper(Bits docsWithField, LegacyBinaryDocValues values) { - this.docsWithField = docsWithField; - this.values = values; - this.maxDoc = docsWithField.length(); - } - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - docID++; - while (docID < maxDoc) { - if (docsWithField.get(docID)) { - return docID; - } - docID++; - } - docID = NO_MORE_DOCS; - return NO_MORE_DOCS; - } - - @Override - public int advance(int target) { - if (target < docID) { - throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target); - } - if (target == NO_MORE_DOCS) { - this.docID = NO_MORE_DOCS; - } else { - this.docID = target - 1; - nextDoc(); - } - return docID; - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - return docsWithField.get(target); - } - - @Override - public long cost() { - return 0; - } - - @Override - public BytesRef binaryValue() { - return values.get(docID); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyDocValuesIterables.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyDocValuesIterables.java deleted file mode 100644 index 5a9b1bb252308..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyDocValuesIterables.java +++ /dev/null @@ -1,539 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.codecs.NormsProducer; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BytesRef; - -import java.io.IOException; -import java.util.Iterator; - -/** Bridge helper methods for legacy codecs to map sorted doc values to iterables. */ - -public class LegacyDocValuesIterables { - - private LegacyDocValuesIterables() { - // no - } - - /** Converts {@link SortedDocValues} into an {@code Iterable<BytesRef>} for all the values. - * - * @deprecated Consume {@link SortedDocValues} instead. */ - @Deprecated - public static Iterable valuesIterable(final SortedDocValues values) { - return new Iterable() { - @Override - public Iterator iterator() { - return new Iterator() { - private int nextOrd; - - @Override - public boolean hasNext() { - return nextOrd < values.getValueCount(); - } - - @Override - public BytesRef next() { - try { - return values.lookupOrd(nextOrd++); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }; - } - }; - } - - /** Converts {@link SortedSetDocValues} into an {@code Iterable<BytesRef>} for all the values. - * - * @deprecated Consume {@link SortedSetDocValues} instead. */ - @Deprecated - public static Iterable valuesIterable(final SortedSetDocValues values) { - return new Iterable() { - @Override - public Iterator iterator() { - return new Iterator() { - private long nextOrd; - - @Override - public boolean hasNext() { - return nextOrd < values.getValueCount(); - } - - @Override - public BytesRef next() { - try { - return values.lookupOrd(nextOrd++); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }; - } - }; - } - - /** Converts {@link SortedDocValues} into the ord for each document as an {@code Iterable<Number>}. - * - * @deprecated Consume {@link SortedDocValues} instead. */ - @Deprecated - public static Iterable sortedOrdIterable(final DocValuesProducer valuesProducer, FieldInfo fieldInfo, int maxDoc) { - return new Iterable() { - @Override - public Iterator iterator() { - - final SortedDocValues values; - try { - values = valuesProducer.getSorted(fieldInfo); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - - return new Iterator() { - private int nextDocID; - - @Override - public boolean hasNext() { - return nextDocID < maxDoc; - } - - @Override - public Number next() { - try { - if (nextDocID > values.docID()) { - values.nextDoc(); - } - int result; - if (nextDocID == values.docID()) { - result = values.ordValue(); - } else { - result = -1; - } - nextDocID++; - return result; - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - }; - } - }; - } - - /** Converts number-of-ords per document from {@link SortedSetDocValues} into {@code Iterable<Number>}. - * - * @deprecated Consume {@link SortedSetDocValues} instead. */ - @Deprecated - public static Iterable sortedSetOrdCountIterable( - final DocValuesProducer valuesProducer, - final FieldInfo fieldInfo, - final int maxDoc - ) { - - return new Iterable() { - - @Override - public Iterator iterator() { - - final SortedSetDocValues values; - try { - values = valuesProducer.getSortedSet(fieldInfo); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - - return new Iterator() { - private int nextDocID; - private int ordCount; - - @Override - public boolean hasNext() { - return nextDocID < maxDoc; - } - - @Override - public Number next() { - try { - if (nextDocID > values.docID()) { - if (values.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { - ordCount = 0; - while (values.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) { - ordCount++; - } - } - } - int result; - if (nextDocID == values.docID()) { - result = ordCount; - } else { - result = 0; - } - nextDocID++; - return result; - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - }; - } - }; - } - - /** Converts all concatenated ords (in docID order) from {@link SortedSetDocValues} into {@code Iterable<Number>}. - * - * @deprecated Consume {@link SortedSetDocValues} instead. */ - @Deprecated - public static Iterable sortedSetOrdsIterable(final DocValuesProducer valuesProducer, final FieldInfo fieldInfo) { - - return new Iterable() { - - @Override - public Iterator iterator() { - - final SortedSetDocValues values; - try { - values = valuesProducer.getSortedSet(fieldInfo); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - - return new Iterator() { - private boolean nextIsSet; - private long nextOrd; - - private void setNext() { - try { - if (nextIsSet == false) { - if (values.docID() == -1) { - values.nextDoc(); - } - while (true) { - if (values.docID() == DocIdSetIterator.NO_MORE_DOCS) { - nextOrd = -1; - break; - } - nextOrd = values.nextOrd(); - if (nextOrd != -1) { - break; - } - values.nextDoc(); - } - nextIsSet = true; - } - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - - @Override - public boolean hasNext() { - setNext(); - return nextOrd != -1; - } - - @Override - public Number next() { - setNext(); - assert nextOrd != -1; - nextIsSet = false; - return nextOrd; - } - }; - } - }; - } - - /** Converts number-of-values per document from {@link SortedNumericDocValues} into {@code Iterable<Number>}. - * - * @deprecated Consume {@link SortedDocValues} instead. */ - @Deprecated - public static Iterable sortedNumericToDocCount(final DocValuesProducer valuesProducer, final FieldInfo fieldInfo, int maxDoc) { - return new Iterable() { - - @Override - public Iterator iterator() { - - final SortedNumericDocValues values; - try { - values = valuesProducer.getSortedNumeric(fieldInfo); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - - return new Iterator() { - private int nextDocID; - - @Override - public boolean hasNext() { - return nextDocID < maxDoc; - } - - @Override - public Number next() { - try { - if (nextDocID > values.docID()) { - values.nextDoc(); - } - int result; - if (nextDocID == values.docID()) { - result = values.docValueCount(); - } else { - result = 0; - } - nextDocID++; - return result; - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - }; - } - }; - } - - /** Converts all concatenated values (in docID order) from {@link SortedNumericDocValues} into {@code Iterable<Number>}. - * - * @deprecated Consume {@link SortedDocValues} instead. */ - @Deprecated - public static Iterable sortedNumericToValues(final DocValuesProducer valuesProducer, final FieldInfo fieldInfo) { - return new Iterable() { - - @Override - public Iterator iterator() { - - final SortedNumericDocValues values; - try { - values = valuesProducer.getSortedNumeric(fieldInfo); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - - return new Iterator() { - private boolean nextIsSet; - private int nextCount; - private int upto; - private long nextValue; - - private void setNext() { - try { - if (nextIsSet == false) { - if (upto == nextCount) { - values.nextDoc(); - if (values.docID() == DocIdSetIterator.NO_MORE_DOCS) { - nextCount = 0; - nextIsSet = false; - return; - } else { - nextCount = values.docValueCount(); - } - upto = 0; - } - nextValue = values.nextValue(); - upto++; - nextIsSet = true; - } - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - - @Override - public boolean hasNext() { - setNext(); - return nextCount != 0; - } - - @Override - public Number next() { - setNext(); - assert nextCount != 0; - nextIsSet = false; - return nextValue; - } - }; - } - }; - } - - /** Converts norms into {@code Iterable<Number>}. - * - * @deprecated Consume {@link NumericDocValues} instead. */ - @Deprecated - public static Iterable normsIterable(final FieldInfo field, final NormsProducer normsProducer, final int maxDoc) { - - return new Iterable() { - - @Override - public Iterator iterator() { - - final NumericDocValues values; - try { - values = normsProducer.getNorms(field); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - - return new Iterator() { - private int docIDUpto = -1; - - @Override - public boolean hasNext() { - return docIDUpto + 1 < maxDoc; - } - - @Override - public Number next() { - docIDUpto++; - if (docIDUpto > values.docID()) { - try { - values.nextDoc(); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - Number result; - if (docIDUpto == values.docID()) { - try { - result = values.longValue(); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } else { - // Unlike NumericDocValues, norms used to return 0 for missing values: - result = 0; - } - return result; - } - }; - } - }; - } - - /** Converts values from {@link BinaryDocValues} into {@code Iterable<BytesRef>}. - * - * @deprecated Consume {@link BinaryDocValues} instead. */ - @Deprecated - public static Iterable binaryIterable(final FieldInfo field, final DocValuesProducer valuesProducer, final int maxDoc) { - return new Iterable() { - @Override - public Iterator iterator() { - - final BinaryDocValues values; - try { - values = valuesProducer.getBinary(field); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - - return new Iterator() { - private int docIDUpto = -1; - - @Override - public boolean hasNext() { - return docIDUpto + 1 < maxDoc; - } - - @Override - public BytesRef next() { - docIDUpto++; - if (docIDUpto > values.docID()) { - try { - values.nextDoc(); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - BytesRef result; - if (docIDUpto == values.docID()) { - try { - result = values.binaryValue(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else { - result = null; - } - return result; - } - }; - } - }; - } - - /** Converts values from {@link NumericDocValues} into {@code Iterable<Number>}. - * - * @deprecated Consume {@link NumericDocValues} instead. */ - @Deprecated - public static Iterable numericIterable(final FieldInfo field, final DocValuesProducer valuesProducer, final int maxDoc) { - return new Iterable() { - @Override - public Iterator iterator() { - - final NumericDocValues values; - try { - values = valuesProducer.getNumeric(field); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - - return new Iterator() { - private int docIDUpto = -1; - - @Override - public boolean hasNext() { - return docIDUpto + 1 < maxDoc; - } - - @Override - public Number next() { - docIDUpto++; - if (docIDUpto > values.docID()) { - try { - values.nextDoc(); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - Number result; - if (docIDUpto == values.docID()) { - try { - result = values.longValue(); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } else { - result = null; - } - return result; - } - }; - } - }; - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyNumericDocValues.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyNumericDocValues.java deleted file mode 100644 index 252bc152ffdd2..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyNumericDocValues.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.NumericDocValues; - -/** - * A per-document numeric value. - * - * @deprecated Use {@link NumericDocValues} instead. - */ -@Deprecated -public abstract class LegacyNumericDocValues { - - /** Sole constructor. (For invocation by subclass - * constructors, typically implicit.) */ - protected LegacyNumericDocValues() {} - - /** - * Returns the numeric value for the specified document ID. - * @param docID document ID to lookup - * @return numeric value - */ - public abstract long get(int docID); -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyNumericDocValuesWrapper.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyNumericDocValuesWrapper.java deleted file mode 100644 index f677c7b39bac7..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyNumericDocValuesWrapper.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.util.Bits; - -import java.io.IOException; - -/** - * Wraps a {@link LegacyNumericDocValues} into a {@link NumericDocValues}. - * - * @deprecated Implement {@link NumericDocValues} directly. - */ -@Deprecated -public final class LegacyNumericDocValuesWrapper extends NumericDocValues { - private final Bits docsWithField; - private final LegacyNumericDocValues values; - private final int maxDoc; - private int docID = -1; - private long value; - - public LegacyNumericDocValuesWrapper(Bits docsWithField, LegacyNumericDocValues values) { - this.docsWithField = docsWithField; - this.values = values; - this.maxDoc = docsWithField.length(); - } - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - docID++; - while (docID < maxDoc) { - value = values.get(docID); - if (value != 0 || docsWithField.get(docID)) { - return docID; - } - docID++; - } - docID = NO_MORE_DOCS; - return NO_MORE_DOCS; - } - - @Override - public int advance(int target) { - assert target >= docID : "target=" + target + " docID=" + docID; - if (target == NO_MORE_DOCS) { - this.docID = NO_MORE_DOCS; - } else { - this.docID = target - 1; - nextDoc(); - } - return docID; - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - value = values.get(docID); - return value != 0 || docsWithField.get(docID); - } - - @Override - public long cost() { - // TODO - return 0; - } - - @Override - public long longValue() { - return value; - } - - @Override - public String toString() { - return "LegacyNumericDocValuesWrapper(" + values + ")"; - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedDocValues.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedDocValues.java deleted file mode 100644 index 5effaa9284540..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedDocValues.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.util.BytesRef; - -/** - * A per-document byte[] with presorted values. - *

- * Per-Document values in a SortedDocValues are deduplicated, dereferenced, - * and sorted into a dictionary of unique values. A pointer to the - * dictionary value (ordinal) can be retrieved for each document. Ordinals - * are dense and in increasing sorted order. - * - * @deprecated Use {@link SortedDocValues} instead. - */ -@Deprecated -public abstract class LegacySortedDocValues extends LegacyBinaryDocValues { - - /** Sole constructor. (For invocation by subclass - * constructors, typically implicit.) */ - protected LegacySortedDocValues() {} - - /** - * Returns the ordinal for the specified docID. - * @param docID document ID to lookup - * @return ordinal for the document: this is dense, starts at 0, then - * increments by 1 for the next value in sorted order. Note that - * missing values are indicated by -1. - */ - public abstract int getOrd(int docID); - - /** Retrieves the value for the specified ordinal. The returned - * {@link BytesRef} may be re-used across calls to {@link #lookupOrd(int)} - * so make sure to {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want - * to keep it around. - * @param ord ordinal to lookup (must be >= 0 and < {@link #getValueCount()}) - * @see #getOrd(int) - */ - public abstract BytesRef lookupOrd(int ord); - - /** - * Returns the number of unique values. - * @return number of unique values in this SortedDocValues. This is - * also equivalent to one plus the maximum ordinal. - */ - public abstract int getValueCount(); - - private final BytesRef empty = new BytesRef(); - - @Override - public BytesRef get(int docID) { - int ord = getOrd(docID); - if (ord == -1) { - return empty; - } else { - return lookupOrd(ord); - } - } - - /** If {@code key} exists, returns its ordinal, else - * returns {@code -insertionPoint-1}, like {@code - * Arrays.binarySearch}. - * - * @param key Key to look up - **/ - public int lookupTerm(BytesRef key) { - int low = 0; - int high = getValueCount() - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - final BytesRef term = lookupOrd(mid); - int cmp = term.compareTo(key); - - if (cmp < 0) { - low = mid + 1; - } else if (cmp > 0) { - high = mid - 1; - } else { - return mid; // key found - } - } - - return -(low + 1); // key not found. - } - - /** - * Returns a {@link TermsEnum} over the values. - * The enum supports {@link TermsEnum#ord()} and {@link TermsEnum#seekExact(long)}. - */ - public TermsEnum termsEnum() { - throw new UnsupportedOperationException(); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedDocValuesWrapper.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedDocValuesWrapper.java deleted file mode 100644 index 592ae3846885a..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedDocValuesWrapper.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.util.BytesRef; - -import java.io.IOException; - -/** - * Wraps a {@link LegacySortedDocValues} into a {@link SortedDocValues}. - * - * @deprecated Implement {@link SortedDocValues} directly. - */ -@Deprecated -public final class LegacySortedDocValuesWrapper extends SortedDocValues { - private final LegacySortedDocValues values; - private final int maxDoc; - private int docID = -1; - private int ord; - - public LegacySortedDocValuesWrapper(LegacySortedDocValues values, int maxDoc) { - this.values = values; - this.maxDoc = maxDoc; - } - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - assert docID != NO_MORE_DOCS; - docID++; - while (docID < maxDoc) { - ord = values.getOrd(docID); - if (ord != -1) { - return docID; - } - docID++; - } - docID = NO_MORE_DOCS; - return NO_MORE_DOCS; - } - - @Override - public int advance(int target) { - if (target < docID) { - throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target); - } - if (target >= maxDoc) { - this.docID = NO_MORE_DOCS; - } else { - this.docID = target - 1; - nextDoc(); - } - return docID; - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - ord = values.getOrd(docID); - return ord != -1; - } - - @Override - public long cost() { - return 0; - } - - @Override - public int ordValue() { - return ord; - } - - @Override - public BytesRef lookupOrd(int ord) { - return values.lookupOrd(ord); - } - - @Override - public int getValueCount() { - return values.getValueCount(); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedNumericDocValues.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedNumericDocValues.java deleted file mode 100644 index fd528e8a6cddf..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedNumericDocValues.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.SortedNumericDocValues; - -/** - * A list of per-document numeric values, sorted - * according to {@link Long#compare(long, long)}. - * - * @deprecated Use {@link SortedNumericDocValues} instead. - */ -@Deprecated -public abstract class LegacySortedNumericDocValues { - - /** Sole constructor. (For invocation by subclass - * constructors, typically implicit.) */ - protected LegacySortedNumericDocValues() {} - - /** - * Positions to the specified document - */ - public abstract void setDocument(int doc); - - /** - * Retrieve the value for the current document at the specified index. - * An index ranges from {@code 0} to {@code count()-1}. - */ - public abstract long valueAt(int index); - - /** - * Retrieves the count of values for the current document. - * This may be zero if a document has no values. - */ - public abstract int count(); -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedNumericDocValuesWrapper.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedNumericDocValuesWrapper.java deleted file mode 100644 index af6b38daf3dbc..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedNumericDocValuesWrapper.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.SortedNumericDocValues; - -import java.io.IOException; - -/** - * Wraps a {@link LegacySortedNumericDocValues} into a {@link SortedNumericDocValues}. - * - * @deprecated Implement {@link SortedNumericDocValues} directly. - */ -@Deprecated -public final class LegacySortedNumericDocValuesWrapper extends SortedNumericDocValues { - private final LegacySortedNumericDocValues values; - private final int maxDoc; - private int docID = -1; - private int upto; - - public LegacySortedNumericDocValuesWrapper(LegacySortedNumericDocValues values, int maxDoc) { - this.values = values; - this.maxDoc = maxDoc; - } - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - assert docID != NO_MORE_DOCS; - while (true) { - docID++; - if (docID == maxDoc) { - docID = NO_MORE_DOCS; - break; - } - values.setDocument(docID); - if (values.count() != 0) { - break; - } - } - upto = 0; - return docID; - } - - @Override - public int advance(int target) { - if (target < docID) { - throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target); - } - if (target >= maxDoc) { - docID = NO_MORE_DOCS; - } else { - docID = target - 1; - nextDoc(); - } - return docID; - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - values.setDocument(docID); - upto = 0; - return values.count() != 0; - } - - @Override - public long cost() { - return 0; - } - - @Override - public long nextValue() { - return values.valueAt(upto++); - } - - @Override - public int docValueCount() { - return values.count(); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValues.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValues.java deleted file mode 100644 index 3af500bcf166e..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValues.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.util.BytesRef; - -import java.io.IOException; - -/** - * A per-document set of presorted byte[] values. - *

- * Per-Document values in a SortedDocValues are deduplicated, dereferenced, - * and sorted into a dictionary of unique values. A pointer to the - * dictionary value (ordinal) can be retrieved for each document. Ordinals - * are dense and in increasing sorted order. - * - * @deprecated Use {@link SortedSetDocValues} instead. - */ -@Deprecated -public abstract class LegacySortedSetDocValues { - - /** Sole constructor. (For invocation by subclass - * constructors, typically implicit.) */ - protected LegacySortedSetDocValues() {} - - /** When returned by {@link #nextOrd()} it means there are no more - * ordinals for the document. - */ - public static final long NO_MORE_ORDS = -1; - - /** - * Returns the next ordinal for the current document (previously - * set by {@link #setDocument(int)}. - * @return next ordinal for the document, or {@link #NO_MORE_ORDS}. - * ordinals are dense, start at 0, then increment by 1 for - * the next value in sorted order. - */ - public abstract long nextOrd(); - - /** - * Sets iteration to the specified docID - * @param docID document ID - */ - public abstract void setDocument(int docID); - - /** Retrieves the value for the specified ordinal. The returned - * {@link BytesRef} may be re-used across calls to lookupOrd so make sure to - * {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want to keep it - * around. - * @param ord ordinal to lookup - * @see #nextOrd - */ - public abstract BytesRef lookupOrd(long ord); - - /** - * Returns the number of unique values. - * @return number of unique values in this SortedDocValues. This is - * also equivalent to one plus the maximum ordinal. - */ - public abstract long getValueCount(); - - /** If {@code key} exists, returns its ordinal, else - * returns {@code -insertionPoint-1}, like {@code - * Arrays.binarySearch}. - * - * @param key Key to look up - **/ - public long lookupTerm(BytesRef key) { - long low = 0; - long high = getValueCount() - 1; - - while (low <= high) { - long mid = (low + high) >>> 1; - final BytesRef term = lookupOrd(mid); - int cmp = term.compareTo(key); - - if (cmp < 0) { - low = mid + 1; - } else if (cmp > 0) { - high = mid - 1; - } else { - return mid; // key found - } - } - - return -(low + 1); // key not found. - } - - /** - * Returns a {@link TermsEnum} over the values. - * The enum supports {@link TermsEnum#ord()} and {@link TermsEnum#seekExact(long)}. - */ - public TermsEnum termsEnum() throws IOException { - throw new UnsupportedOperationException(); - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java deleted file mode 100644 index 272929346448d..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.index; - -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.BytesRef; - -import java.io.IOException; - -/** - * Wraps a {@link LegacySortedSetDocValues} into a {@link SortedSetDocValues}. - * - * @deprecated Implement {@link SortedSetDocValues} directly. - */ -@Deprecated -public final class LegacySortedSetDocValuesWrapper extends SortedSetDocValues { - private final LegacySortedSetDocValues values; - private final int maxDoc; - private int docID = -1; - private long ord; - - public LegacySortedSetDocValuesWrapper(LegacySortedSetDocValues values, int maxDoc) { - this.values = values; - this.maxDoc = maxDoc; - } - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - assert docID != NO_MORE_DOCS; - docID++; - while (docID < maxDoc) { - values.setDocument(docID); - ord = values.nextOrd(); - if (ord != NO_MORE_ORDS) { - return docID; - } - docID++; - } - docID = NO_MORE_DOCS; - return NO_MORE_DOCS; - } - - @Override - public int advance(int target) { - if (target < docID) { - throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target); - } - if (target >= maxDoc) { - this.docID = NO_MORE_DOCS; - } else { - this.docID = target - 1; - nextDoc(); - } - return docID; - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - values.setDocument(docID); - ord = values.nextOrd(); - return ord != NO_MORE_ORDS; - } - - @Override - public long cost() { - return 0; - } - - @Override - public long nextOrd() { - long result = ord; - if (result != NO_MORE_ORDS) { - ord = values.nextOrd(); - } - return result; - } - - @Override - public BytesRef lookupOrd(long ord) { - return values.lookupOrd((int) ord); - } - - @Override - public long getValueCount() { - return values.getValueCount(); - } - - @Override - public String toString() { - return "LegacySortedSetDocValuesWrapper(" + values + ")"; - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java deleted file mode 100644 index 11966fb396724..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene50; - -import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -/** - * Lucene 5.0 Field Infos format. - * @deprecated - */ -@Deprecated -public final class Lucene50FieldInfosFormat extends FieldInfosFormat { - - public Lucene50FieldInfosFormat() {} - - @Override - public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException { - final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); - try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(directory, fileName, context)) { - Throwable priorE = null; - FieldInfo infos[] = null; - try { - CodecUtil.checkIndexHeader( - input, - Lucene50FieldInfosFormat.CODEC_NAME, - Lucene50FieldInfosFormat.FORMAT_START, - Lucene50FieldInfosFormat.FORMAT_CURRENT, - segmentInfo.getId(), - segmentSuffix - ); - - final int size = input.readVInt(); // read in the size - infos = new FieldInfo[size]; - - // previous field's attribute map, we share when possible: - Map lastAttributes = Collections.emptyMap(); - - for (int i = 0; i < size; i++) { - String name = input.readString(); - final int fieldNumber = input.readVInt(); - if (fieldNumber < 0) { - throw new CorruptIndexException("invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input); - } - byte bits = input.readByte(); - boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; - boolean omitNorms = (bits & OMIT_NORMS) != 0; - boolean storePayloads = (bits & STORE_PAYLOADS) != 0; - - final IndexOptions indexOptions = getIndexOptions(input, input.readByte()); - - // DV Types are packed in one byte - final DocValuesType docValuesType = getDocValuesType(input, input.readByte()); - final long dvGen = input.readLong(); - Map attributes = input.readMapOfStrings(); - - // just use the last field's map if its the same - if (attributes.equals(lastAttributes)) { - attributes = lastAttributes; - } - lastAttributes = attributes; - try { - infos[i] = new FieldInfo( - name, - fieldNumber, - storeTermVector, - omitNorms, - storePayloads, - indexOptions, - docValuesType, - dvGen, - attributes, - 0, - 0, - 0, - 0, - VectorSimilarityFunction.EUCLIDEAN, - false - ); - infos[i].checkConsistency(); - } catch (IllegalStateException e) { - throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e); - } - } - } catch (Throwable exception) { - priorE = exception; - } finally { - CodecUtil.checkFooter(input, priorE); - } - return new FieldInfos(infos); - } - } - - private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException { - return switch (b) { - case 0 -> DocValuesType.NONE; - case 1 -> DocValuesType.NUMERIC; - case 2 -> DocValuesType.BINARY; - case 3 -> DocValuesType.SORTED; - case 4 -> DocValuesType.SORTED_SET; - case 5 -> DocValuesType.SORTED_NUMERIC; - default -> throw new CorruptIndexException("invalid docvalues byte: " + b, input); - }; - } - - private static IndexOptions getIndexOptions(IndexInput input, byte b) throws IOException { - return switch (b) { - case 0 -> IndexOptions.NONE; - case 1 -> IndexOptions.DOCS; - case 2 -> IndexOptions.DOCS_AND_FREQS; - case 3 -> IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - case 4 -> IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; - default -> - // BUG - throw new CorruptIndexException("invalid IndexOptions byte: " + b, input); - }; - } - - @Override - public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) { - throw new UnsupportedOperationException(); - } - - /** Extension of field infos */ - static final String EXTENSION = "fnm"; - - // Codec header - static final String CODEC_NAME = "Lucene50FieldInfos"; - static final int FORMAT_SAFE_MAPS = 1; - static final int FORMAT_START = FORMAT_SAFE_MAPS; - static final int FORMAT_CURRENT = FORMAT_SAFE_MAPS; - - // Field flags - static final byte STORE_TERMVECTOR = 0x1; - static final byte OMIT_NORMS = 0x2; - static final byte STORE_PAYLOADS = 0x4; -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java deleted file mode 100644 index cf4437a230c0d..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene50; - -import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.Version; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; - -/** - * Lucene 5.0 Segment info format. - * @deprecated Only for reading old 5.0-6.0 segments - */ -@Deprecated -public class Lucene50SegmentInfoFormat extends SegmentInfoFormat { - - public Lucene50SegmentInfoFormat() {} - - @Override - public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException { - final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene50SegmentInfoFormat.SI_EXTENSION); - try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(dir, fileName, context)) { - Throwable priorE = null; - SegmentInfo si = null; - try { - CodecUtil.checkIndexHeader( - input, - Lucene50SegmentInfoFormat.CODEC_NAME, - Lucene50SegmentInfoFormat.VERSION_START, - Lucene50SegmentInfoFormat.VERSION_CURRENT, - segmentID, - "" - ); - final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt()); - - final int docCount = input.readInt(); - if (docCount < 0) { - throw new CorruptIndexException("invalid docCount: " + docCount, input); - } - final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; - - final Map diagnostics = input.readMapOfStrings(); - final Set files = input.readSetOfStrings(); - final Map attributes = input.readMapOfStrings(); - - si = new SegmentInfo(dir, version, null, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, null); - si.setFiles(files); - } catch (Throwable exception) { - priorE = exception; - } finally { - CodecUtil.checkFooter(input, priorE); - } - return si; - } - } - - @Override - public void write(Directory dir, SegmentInfo si, IOContext ioContext) { - throw new UnsupportedOperationException("this codec can only be used for reading"); - } - - /** File extension used to store {@link SegmentInfo}. */ - public static final String SI_EXTENSION = "si"; - static final String CODEC_NAME = "Lucene50SegmentInfo"; - static final int VERSION_SAFE_MAPS = 1; - static final int VERSION_START = VERSION_SAFE_MAPS; - static final int VERSION_CURRENT = VERSION_SAFE_MAPS; -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/LegacyStringHelper.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/LegacyStringHelper.java deleted file mode 100644 index 50e5cde04ead3..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/LegacyStringHelper.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene54; - -import org.apache.lucene.util.BytesRef; - -/** - * Legacy methods for manipulating strings. - * - * @lucene.internal - * @deprecated This is only used for backwards compatibility codecs (they - * don't work with the Java9-based replacement methods). - */ -@Deprecated -abstract class LegacyStringHelper { - - /** - * Compares two {@link BytesRef}, element by element, and returns the - * number of elements common to both arrays (from the start of each). - * - * @param left The first {@link BytesRef} to compare - * @param right The second {@link BytesRef} to compare - * @return The number of common elements (from the start of each). - */ - public static int bytesDifference(BytesRef left, BytesRef right) { - int len = left.length < right.length ? left.length : right.length; - final byte[] bytesLeft = left.bytes; - final int offLeft = left.offset; - byte[] bytesRight = right.bytes; - final int offRight = right.offset; - for (int i = 0; i < len; i++) - if (bytesLeft[i + offLeft] != bytesRight[i + offRight]) return i; - return len; - } - - /** - * Returns the length of {@code currentTerm} needed for use as a sort key. - * so that {@link BytesRef#compareTo(BytesRef)} still returns the same result. - * This method assumes currentTerm comes after priorTerm. - */ - public static int sortKeyLength(final BytesRef priorTerm, final BytesRef currentTerm) { - final int currentTermOffset = currentTerm.offset; - final int priorTermOffset = priorTerm.offset; - final int limit = Math.min(priorTerm.length, currentTerm.length); - for (int i = 0; i < limit; i++) { - if (priorTerm.bytes[priorTermOffset + i] != currentTerm.bytes[currentTermOffset + i]) { - return i + 1; - } - } - return Math.min(1 + priorTerm.length, currentTerm.length); - } - - private LegacyStringHelper() {} - -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesConsumer.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesConsumer.java deleted file mode 100644 index fe8d24262f056..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesConsumer.java +++ /dev/null @@ -1,842 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene54; - -import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicWriter; -import org.apache.lucene.backward_codecs.packed.LegacyDirectWriter; -import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.store.ByteBuffersDataOutput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LongsRef; -import org.apache.lucene.util.MathUtil; -import org.apache.lucene.util.PagedBytes; -import org.apache.lucene.util.packed.MonotonicBlockPackedWriter; -import org.apache.lucene.util.packed.PackedInts; -import org.elasticsearch.core.internal.io.IOUtils; -import org.elasticsearch.xpack.lucene.bwc.codecs.index.LegacyDocValuesIterables; - -import java.io.Closeable; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.stream.StreamSupport; - -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.ALL_LIVE; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.ALL_MISSING; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.BINARY_FIXED_UNCOMPRESSED; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.BINARY_PREFIX_COMPRESSED; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.BINARY_VARIABLE_UNCOMPRESSED; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.CONST_COMPRESSED; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.DELTA_COMPRESSED; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.GCD_COMPRESSED; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.INTERVAL_COUNT; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.INTERVAL_MASK; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.INTERVAL_SHIFT; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.MONOTONIC_BLOCK_SIZE; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.MONOTONIC_COMPRESSED; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.REVERSE_INTERVAL_COUNT; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.REVERSE_INTERVAL_MASK; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.SORTED_SET_TABLE; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.SORTED_SINGLE_VALUED; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.SORTED_WITH_ADDRESSES; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.SPARSE_COMPRESSED; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat.TABLE_COMPRESSED; - -/** writer for {@link Lucene54DocValuesFormat} */ -final class Lucene54DocValuesConsumer extends DocValuesConsumer implements Closeable { - - enum NumberType { - /** Dense ordinals */ - ORDINAL, - /** Random long values */ - VALUE; - } - - IndexOutput data, meta; - final int maxDoc; - - /** expert: Creates a new writer */ - Lucene54DocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) - throws IOException { - boolean success = false; - try { - String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); - data = EndiannessReverserUtil.createOutput(state.directory, dataName, state.context); - CodecUtil.writeIndexHeader( - data, - dataCodec, - Lucene54DocValuesFormat.VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); - meta = EndiannessReverserUtil.createOutput(state.directory, metaName, state.context); - CodecUtil.writeIndexHeader( - meta, - metaCodec, - Lucene54DocValuesFormat.VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - maxDoc = state.segmentInfo.maxDoc(); - success = true; - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(this); - } - } - } - - @Override - public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { - addNumericField(field, LegacyDocValuesIterables.numericIterable(field, valuesProducer, maxDoc), NumberType.VALUE); - } - - void addNumericField(FieldInfo field, Iterable values, NumberType numberType) throws IOException { - long count = 0; - long minValue = Long.MAX_VALUE; - long maxValue = Long.MIN_VALUE; - long gcd = 0; - long missingCount = 0; - long zeroCount = 0; - // TODO: more efficient? - HashSet uniqueValues = null; - long missingOrdCount = 0; - if (numberType == NumberType.VALUE) { - uniqueValues = new HashSet<>(); - - for (Number nv : values) { - final long v; - if (nv == null) { - v = 0; - missingCount++; - zeroCount++; - } else { - v = nv.longValue(); - if (v == 0) { - zeroCount++; - } - } - - if (gcd != 1) { - if (v < Long.MIN_VALUE / 2 || v > Long.MAX_VALUE / 2) { - // in that case v - minValue might overflow and make the GCD computation return - // wrong results. Since these extreme values are unlikely, we just discard - // GCD computation for them - gcd = 1; - } else if (count != 0) { // minValue needs to be set first - gcd = MathUtil.gcd(gcd, v - minValue); - } - } - - minValue = Math.min(minValue, v); - maxValue = Math.max(maxValue, v); - - if (uniqueValues != null) { - if (uniqueValues.add(v)) { - if (uniqueValues.size() > 256) { - uniqueValues = null; - } - } - } - - ++count; - } - } else { - for (Number nv : values) { - long v = nv.longValue(); - if (v == -1L) { - missingOrdCount++; - } - minValue = Math.min(minValue, v); - maxValue = Math.max(maxValue, v); - ++count; - } - } - - final long delta = maxValue - minValue; - final int deltaBitsRequired = LegacyDirectWriter.unsignedBitsRequired(delta); - final int tableBitsRequired = uniqueValues == null ? Integer.MAX_VALUE : LegacyDirectWriter.bitsRequired(uniqueValues.size() - 1); - - final boolean sparse; // 1% of docs or less have a value - switch (numberType) { - case VALUE: - sparse = (double) missingCount / count >= 0.99; - break; - case ORDINAL: - sparse = (double) missingOrdCount / count >= 0.99; - break; - default: - throw new AssertionError(); - } - - final int format; - if (uniqueValues != null - && count <= Integer.MAX_VALUE - && (uniqueValues.size() == 1 || (uniqueValues.size() == 2 && missingCount > 0 && zeroCount == missingCount))) { - // either one unique value C or two unique values: "missing" and C - format = CONST_COMPRESSED; - } else if (sparse && count >= 1024) { - // require at least 1024 docs to avoid flipping back and forth when doing NRT search - format = SPARSE_COMPRESSED; - } else if (uniqueValues != null && tableBitsRequired < deltaBitsRequired) { - format = TABLE_COMPRESSED; - } else if (gcd != 0 && gcd != 1) { - final long gcdDelta = (maxValue - minValue) / gcd; - final long gcdBitsRequired = LegacyDirectWriter.unsignedBitsRequired(gcdDelta); - format = gcdBitsRequired < deltaBitsRequired ? GCD_COMPRESSED : DELTA_COMPRESSED; - } else { - format = DELTA_COMPRESSED; - } - meta.writeVInt(field.number); - meta.writeByte(Lucene54DocValuesFormat.NUMERIC); - meta.writeVInt(format); - if (format == SPARSE_COMPRESSED) { - meta.writeLong(data.getFilePointer()); - final long numDocsWithValue; - switch (numberType) { - case VALUE: - numDocsWithValue = count - missingCount; - break; - case ORDINAL: - numDocsWithValue = count - missingOrdCount; - break; - default: - throw new AssertionError(); - } - final long maxDoc = writeSparseMissingBitset(values, numberType, numDocsWithValue); - assert maxDoc == count; - } else if (missingCount == 0) { - meta.writeLong(ALL_LIVE); - } else if (missingCount == count) { - meta.writeLong(ALL_MISSING); - } else { - meta.writeLong(data.getFilePointer()); - writeMissingBitset(values); - } - meta.writeLong(data.getFilePointer()); - meta.writeVLong(count); - - switch (format) { - case CONST_COMPRESSED: - // write the constant (nonzero value in the n=2 case, singleton value otherwise) - meta.writeLong(minValue < 0 ? Collections.min(uniqueValues) : Collections.max(uniqueValues)); - break; - case GCD_COMPRESSED: - meta.writeLong(minValue); - meta.writeLong(gcd); - final long maxDelta = (maxValue - minValue) / gcd; - final int bits = LegacyDirectWriter.unsignedBitsRequired(maxDelta); - meta.writeVInt(bits); - final LegacyDirectWriter quotientWriter = LegacyDirectWriter.getInstance(data, count, bits); - for (Number nv : values) { - long value = nv == null ? 0 : nv.longValue(); - quotientWriter.add((value - minValue) / gcd); - } - quotientWriter.finish(); - break; - case DELTA_COMPRESSED: - final long minDelta = delta < 0 ? 0 : minValue; - meta.writeLong(minDelta); - meta.writeVInt(deltaBitsRequired); - final LegacyDirectWriter writer = LegacyDirectWriter.getInstance(data, count, deltaBitsRequired); - for (Number nv : values) { - long v = nv == null ? 0 : nv.longValue(); - writer.add(v - minDelta); - } - writer.finish(); - break; - case TABLE_COMPRESSED: - final Long[] decode = uniqueValues.toArray(new Long[uniqueValues.size()]); - Arrays.sort(decode); - final HashMap encode = new HashMap<>(); - meta.writeVInt(decode.length); - for (int i = 0; i < decode.length; i++) { - meta.writeLong(decode[i]); - encode.put(decode[i], i); - } - meta.writeVInt(tableBitsRequired); - final LegacyDirectWriter ordsWriter = LegacyDirectWriter.getInstance(data, count, tableBitsRequired); - for (Number nv : values) { - ordsWriter.add(encode.get(nv == null ? 0 : nv.longValue())); - } - ordsWriter.finish(); - break; - case SPARSE_COMPRESSED: - final Iterable filteredMissingValues; - switch (numberType) { - case VALUE: - meta.writeByte((byte) 0); - filteredMissingValues = new Iterable() { - @Override - public Iterator iterator() { - return StreamSupport.stream(values.spliterator(), false).filter(value -> value != null).iterator(); - } - }; - break; - case ORDINAL: - meta.writeByte((byte) 1); - filteredMissingValues = new Iterable() { - @Override - public Iterator iterator() { - return StreamSupport.stream(values.spliterator(), false) - .filter(value -> value.longValue() != -1L) - .iterator(); - } - }; - break; - default: - throw new AssertionError(); - } - // Write non-missing values as a numeric field - addNumericField(field, filteredMissingValues, numberType); - break; - default: - throw new AssertionError(); - } - meta.writeLong(data.getFilePointer()); - } - - // TODO: in some cases representing missing with minValue-1 wouldn't take up additional space and so on, - // but this is very simple, and algorithms only check this for values of 0 anyway (doesnt slow down normal decode) - void writeMissingBitset(Iterable values) throws IOException { - byte bits = 0; - int count = 0; - for (Object v : values) { - if (count == 8) { - data.writeByte(bits); - count = 0; - bits = 0; - } - if (v != null) { - bits |= 1 << (count & 7); - } - count++; - } - if (count > 0) { - data.writeByte(bits); - } - } - - long writeSparseMissingBitset(Iterable values, NumberType numberType, long numDocsWithValue) throws IOException { - meta.writeVLong(numDocsWithValue); - - // Write doc IDs that have a value - meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); - final LegacyDirectMonotonicWriter docIdsWriter = LegacyDirectMonotonicWriter.getInstance( - meta, - data, - numDocsWithValue, - DIRECT_MONOTONIC_BLOCK_SHIFT - ); - long docID = 0; - for (Number nv : values) { - switch (numberType) { - case VALUE: - if (nv != null) { - docIdsWriter.add(docID); - } - break; - case ORDINAL: - if (nv.longValue() != -1L) { - docIdsWriter.add(docID); - } - break; - default: - throw new AssertionError(); - } - docID++; - } - docIdsWriter.finish(); - return docID; - } - - @Override - public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { - addBinaryField(field, LegacyDocValuesIterables.binaryIterable(field, valuesProducer, maxDoc)); - } - - private void addBinaryField(FieldInfo field, Iterable values) throws IOException { - // write the byte[] data - meta.writeVInt(field.number); - meta.writeByte(Lucene54DocValuesFormat.BINARY); - int minLength = Integer.MAX_VALUE; - int maxLength = Integer.MIN_VALUE; - final long startFP = data.getFilePointer(); - long count = 0; - long missingCount = 0; - for (BytesRef v : values) { - final int length; - if (v == null) { - length = 0; - missingCount++; - } else { - length = v.length; - } - minLength = Math.min(minLength, length); - maxLength = Math.max(maxLength, length); - if (v != null) { - data.writeBytes(v.bytes, v.offset, v.length); - } - count++; - } - meta.writeVInt(minLength == maxLength ? BINARY_FIXED_UNCOMPRESSED : BINARY_VARIABLE_UNCOMPRESSED); - if (missingCount == 0) { - meta.writeLong(ALL_LIVE); - } else if (missingCount == count) { - meta.writeLong(ALL_MISSING); - } else { - meta.writeLong(data.getFilePointer()); - writeMissingBitset(values); - } - meta.writeVInt(minLength); - meta.writeVInt(maxLength); - meta.writeVLong(count); - meta.writeLong(startFP); - - // if minLength == maxLength, it's a fixed-length byte[], we are done (the addresses are implicit) - // otherwise, we need to record the length fields... - if (minLength != maxLength) { - meta.writeLong(data.getFilePointer()); - meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); - - final LegacyDirectMonotonicWriter writer = LegacyDirectMonotonicWriter.getInstance( - meta, - data, - count + 1, - DIRECT_MONOTONIC_BLOCK_SHIFT - ); - long addr = 0; - writer.add(addr); - for (BytesRef v : values) { - if (v != null) { - addr += v.length; - } - writer.add(addr); - } - writer.finish(); - meta.writeLong(data.getFilePointer()); - } - } - - /** expert: writes a value dictionary for a sorted/sortedset field */ - private void addTermsDict(FieldInfo field, final Iterable values) throws IOException { - // first check if it's a "fixed-length" terms dict, and compressibility if so - int minLength = Integer.MAX_VALUE; - int maxLength = Integer.MIN_VALUE; - long numValues = 0; - BytesRefBuilder previousValue = new BytesRefBuilder(); - long prefixSum = 0; // only valid for fixed-width data, as we have a choice there - for (BytesRef v : values) { - minLength = Math.min(minLength, v.length); - maxLength = Math.max(maxLength, v.length); - if (minLength == maxLength) { - int termPosition = (int) (numValues & INTERVAL_MASK); - if (termPosition == 0) { - // first term in block, save it away to compare against the last term later - previousValue.copyBytes(v); - } else if (termPosition == INTERVAL_COUNT - 1) { - // last term in block, accumulate shared prefix against first term - prefixSum += LegacyStringHelper.bytesDifference(previousValue.get(), v); - } - } - numValues++; - } - // for fixed width data, look at the avg(shared prefix) before deciding how to encode: - // prefix compression "costs" worst case 2 bytes per term because we must store suffix lengths. - // so if we share at least 3 bytes on average, always compress. - if (minLength == maxLength && prefixSum <= 3 * (numValues >> INTERVAL_SHIFT)) { - // no index needed: not very compressible, direct addressing by mult - addBinaryField(field, values); - } else if (numValues < REVERSE_INTERVAL_COUNT) { - // low cardinality: waste a few KB of ram, but can't really use fancy index etc - addBinaryField(field, values); - } else { - assert numValues > 0; // we don't have to handle the empty case - // header - meta.writeVInt(field.number); - meta.writeByte(Lucene54DocValuesFormat.BINARY); - meta.writeVInt(BINARY_PREFIX_COMPRESSED); - meta.writeLong(-1L); - // now write the bytes: sharing prefixes within a block - final long startFP = data.getFilePointer(); - // currently, we have to store the delta from expected for every 1/nth term - // we could avoid this, but it's not much and less overall RAM than the previous approach! - ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput(); - MonotonicBlockPackedWriter termAddresses = new MonotonicBlockPackedWriter(addressBuffer, MONOTONIC_BLOCK_SIZE); - // buffers up 16 terms - ByteBuffersDataOutput bytesBuffer = new ByteBuffersDataOutput(); - // buffers up block header - ByteBuffersDataOutput headerBuffer = new ByteBuffersDataOutput(); - BytesRefBuilder lastTerm = new BytesRefBuilder(); - lastTerm.grow(maxLength); - long count = 0; - int suffixDeltas[] = new int[INTERVAL_COUNT]; - for (BytesRef v : values) { - int termPosition = (int) (count & INTERVAL_MASK); - if (termPosition == 0) { - termAddresses.add(data.getFilePointer() - startFP); - // abs-encode first term - headerBuffer.writeVInt(v.length); - headerBuffer.writeBytes(v.bytes, v.offset, v.length); - lastTerm.copyBytes(v); - } else { - // prefix-code: we only share at most 255 characters, to encode the length as a single - // byte and have random access. Larger terms just get less compression. - int sharedPrefix = Math.min(255, LegacyStringHelper.bytesDifference(lastTerm.get(), v)); - bytesBuffer.writeByte((byte) sharedPrefix); - bytesBuffer.writeBytes(v.bytes, v.offset + sharedPrefix, v.length - sharedPrefix); - // we can encode one smaller, because terms are unique. - suffixDeltas[termPosition] = v.length - sharedPrefix - 1; - } - - count++; - // flush block - if ((count & INTERVAL_MASK) == 0) { - flushTermsDictBlock(headerBuffer, bytesBuffer, suffixDeltas); - } - } - // flush trailing crap - int leftover = (int) (count & INTERVAL_MASK); - if (leftover > 0) { - Arrays.fill(suffixDeltas, leftover, suffixDeltas.length, 0); - flushTermsDictBlock(headerBuffer, bytesBuffer, suffixDeltas); - } - final long indexStartFP = data.getFilePointer(); - // write addresses of indexed terms - termAddresses.finish(); - addressBuffer.copyTo(data); - addressBuffer = null; - termAddresses = null; - meta.writeVInt(minLength); - meta.writeVInt(maxLength); - meta.writeVLong(count); - meta.writeLong(startFP); - meta.writeLong(indexStartFP); - meta.writeVInt(PackedInts.VERSION_MONOTONIC_WITHOUT_ZIGZAG); - meta.writeVInt(MONOTONIC_BLOCK_SIZE); - addReverseTermIndex(field, values, maxLength); - } - } - - // writes term dictionary "block" - // first term is absolute encoded as vint length + bytes. - // lengths of subsequent N terms are encoded as either N bytes or N shorts. - // in the double-byte case, the first byte is indicated with -1. - // subsequent terms are encoded as byte suffixLength + bytes. - private void flushTermsDictBlock(ByteBuffersDataOutput headerBuffer, ByteBuffersDataOutput bytesBuffer, int suffixDeltas[]) - throws IOException { - boolean twoByte = false; - for (int i = 1; i < suffixDeltas.length; i++) { - if (suffixDeltas[i] > 254) { - twoByte = true; - } - } - if (twoByte) { - headerBuffer.writeByte((byte) 255); - for (int i = 1; i < suffixDeltas.length; i++) { - headerBuffer.writeShort((short) suffixDeltas[i]); - } - } else { - for (int i = 1; i < suffixDeltas.length; i++) { - headerBuffer.writeByte((byte) suffixDeltas[i]); - } - } - headerBuffer.copyTo(data); - headerBuffer.reset(); - bytesBuffer.copyTo(data); - bytesBuffer.reset(); - } - - // writes reverse term index: used for binary searching a term into a range of 64 blocks - // for every 64 blocks (1024 terms) we store a term, trimming any suffix unnecessary for comparison - // terms are written as a contiguous byte[], but never spanning 2^15 byte boundaries. - private void addReverseTermIndex(FieldInfo field, final Iterable values, int maxLength) throws IOException { - long count = 0; - BytesRefBuilder priorTerm = new BytesRefBuilder(); - priorTerm.grow(maxLength); - BytesRef indexTerm = new BytesRef(); - long startFP = data.getFilePointer(); - PagedBytes pagedBytes = new PagedBytes(15); - MonotonicBlockPackedWriter addresses = new MonotonicBlockPackedWriter(data, MONOTONIC_BLOCK_SIZE); - - for (BytesRef b : values) { - int termPosition = (int) (count & REVERSE_INTERVAL_MASK); - if (termPosition == 0) { - int len = LegacyStringHelper.sortKeyLength(priorTerm.get(), b); - indexTerm.bytes = b.bytes; - indexTerm.offset = b.offset; - indexTerm.length = len; - addresses.add(pagedBytes.copyUsingLengthPrefix(indexTerm)); - } else if (termPosition == REVERSE_INTERVAL_MASK) { - priorTerm.copyBytes(b); - } - count++; - } - addresses.finish(); - long numBytes = pagedBytes.getPointer(); - pagedBytes.freeze(true); - PagedBytes.PagedBytesDataInput in = pagedBytes.getDataInput(); - meta.writeLong(startFP); - data.writeVLong(numBytes); - data.copyBytes(in, numBytes); - } - - @Override - public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { - meta.writeVInt(field.number); - meta.writeByte(Lucene54DocValuesFormat.SORTED); - addTermsDict(field, LegacyDocValuesIterables.valuesIterable(valuesProducer.getSorted(field))); - addNumericField(field, LegacyDocValuesIterables.sortedOrdIterable(valuesProducer, field, maxDoc), NumberType.ORDINAL); - } - - private void addSortedField(FieldInfo field, Iterable values, Iterable ords) throws IOException { - meta.writeVInt(field.number); - meta.writeByte(Lucene54DocValuesFormat.SORTED); - addTermsDict(field, values); - addNumericField(field, ords, NumberType.ORDINAL); - } - - @Override - public void addSortedNumericField(FieldInfo field, final DocValuesProducer valuesProducer) throws IOException { - - final Iterable docToValueCount = LegacyDocValuesIterables.sortedNumericToDocCount(valuesProducer, field, maxDoc); - final Iterable values = LegacyDocValuesIterables.sortedNumericToValues(valuesProducer, field); - - meta.writeVInt(field.number); - meta.writeByte(Lucene54DocValuesFormat.SORTED_NUMERIC); - if (isSingleValued(docToValueCount)) { - meta.writeVInt(SORTED_SINGLE_VALUED); - // The field is single-valued, we can encode it as NUMERIC - addNumericField(field, singletonView(docToValueCount, values, null), NumberType.VALUE); - } else { - final SortedSet uniqueValueSets = uniqueValueSets(docToValueCount, values); - if (uniqueValueSets != null) { - meta.writeVInt(SORTED_SET_TABLE); - - // write the set_id -> values mapping - writeDictionary(uniqueValueSets); - - // write the doc -> set_id as a numeric field - addNumericField(field, docToSetId(uniqueValueSets, docToValueCount, values), NumberType.ORDINAL); - } else { - meta.writeVInt(SORTED_WITH_ADDRESSES); - // write the stream of values as a numeric field - addNumericField(field, values, NumberType.VALUE); - // write the doc -> ord count as a absolute index to the stream - addOrdIndex(field, docToValueCount); - } - } - } - - @Override - public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { - - Iterable values = LegacyDocValuesIterables.valuesIterable(valuesProducer.getSortedSet(field)); - Iterable docToOrdCount = LegacyDocValuesIterables.sortedSetOrdCountIterable(valuesProducer, field, maxDoc); - Iterable ords = LegacyDocValuesIterables.sortedSetOrdsIterable(valuesProducer, field); - - meta.writeVInt(field.number); - meta.writeByte(Lucene54DocValuesFormat.SORTED_SET); - - if (isSingleValued(docToOrdCount)) { - meta.writeVInt(SORTED_SINGLE_VALUED); - // The field is single-valued, we can encode it as SORTED - addSortedField(field, values, singletonView(docToOrdCount, ords, -1L)); - } else { - final SortedSet uniqueValueSets = uniqueValueSets(docToOrdCount, ords); - if (uniqueValueSets != null) { - meta.writeVInt(SORTED_SET_TABLE); - - // write the set_id -> ords mapping - writeDictionary(uniqueValueSets); - - // write the ord -> byte[] as a binary field - addTermsDict(field, values); - - // write the doc -> set_id as a numeric field - addNumericField(field, docToSetId(uniqueValueSets, docToOrdCount, ords), NumberType.ORDINAL); - } else { - meta.writeVInt(SORTED_WITH_ADDRESSES); - - // write the ord -> byte[] as a binary field - addTermsDict(field, values); - - // write the stream of ords as a numeric field - // NOTE: we could return an iterator that delta-encodes these within a doc - addNumericField(field, ords, NumberType.ORDINAL); - - // write the doc -> ord count as a absolute index to the stream - addOrdIndex(field, docToOrdCount); - } - } - } - - private SortedSet uniqueValueSets(Iterable docToValueCount, Iterable values) { - Set uniqueValueSet = new HashSet<>(); - LongsRef docValues = new LongsRef(256); - - Iterator valueCountIterator = docToValueCount.iterator(); - Iterator valueIterator = values.iterator(); - int totalDictSize = 0; - while (valueCountIterator.hasNext()) { - docValues.length = valueCountIterator.next().intValue(); - if (docValues.length > 256) { - return null; - } - for (int i = 0; i < docValues.length; ++i) { - docValues.longs[i] = valueIterator.next().longValue(); - } - if (uniqueValueSet.contains(docValues)) { - continue; - } - totalDictSize += docValues.length; - if (totalDictSize > 256) { - return null; - } - uniqueValueSet.add(new LongsRef(ArrayUtil.copyOfSubArray(docValues.longs, 0, docValues.length), 0, docValues.length)); - } - assert valueIterator.hasNext() == false; - return new TreeSet<>(uniqueValueSet); - } - - private void writeDictionary(SortedSet uniqueValueSets) throws IOException { - int lengthSum = 0; - for (LongsRef longs : uniqueValueSets) { - lengthSum += longs.length; - } - - meta.writeInt(lengthSum); - for (LongsRef valueSet : uniqueValueSets) { - for (int i = 0; i < valueSet.length; ++i) { - meta.writeLong(valueSet.longs[valueSet.offset + i]); - } - } - - meta.writeInt(uniqueValueSets.size()); - for (LongsRef valueSet : uniqueValueSets) { - meta.writeInt(valueSet.length); - } - } - - private Iterable docToSetId(SortedSet uniqueValueSets, Iterable docToValueCount, Iterable values) { - final Map setIds = new HashMap<>(); - int i = 0; - for (LongsRef set : uniqueValueSets) { - setIds.put(set, i++); - } - assert i == uniqueValueSets.size(); - - return new Iterable() { - - @Override - public Iterator iterator() { - final Iterator valueCountIterator = docToValueCount.iterator(); - final Iterator valueIterator = values.iterator(); - final LongsRef docValues = new LongsRef(256); - return new Iterator() { - - @Override - public boolean hasNext() { - return valueCountIterator.hasNext(); - } - - @Override - public Number next() { - docValues.length = valueCountIterator.next().intValue(); - for (int i = 0; i < docValues.length; ++i) { - docValues.longs[i] = valueIterator.next().longValue(); - } - final Integer id = setIds.get(docValues); - assert id != null; - return id; - } - - }; - - } - }; - } - - // writes addressing information as MONOTONIC_COMPRESSED integer - private void addOrdIndex(FieldInfo field, Iterable values) throws IOException { - meta.writeVInt(field.number); - meta.writeByte(Lucene54DocValuesFormat.NUMERIC); - meta.writeVInt(MONOTONIC_COMPRESSED); - meta.writeLong(-1L); - meta.writeLong(data.getFilePointer()); - meta.writeVLong(maxDoc); - meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); - - final LegacyDirectMonotonicWriter writer = LegacyDirectMonotonicWriter.getInstance( - meta, - data, - maxDoc + 1, - DIRECT_MONOTONIC_BLOCK_SHIFT - ); - long addr = 0; - writer.add(addr); - for (Number v : values) { - addr += v.longValue(); - writer.add(addr); - } - writer.finish(); - meta.writeLong(data.getFilePointer()); - } - - @Override - public void close() throws IOException { - boolean success = false; - try { - if (meta != null) { - meta.writeVInt(-1); // write EOF marker - CodecUtil.writeFooter(meta); // write checksum - } - if (data != null) { - CodecUtil.writeFooter(data); // write checksum - } - success = true; - } finally { - if (success) { - IOUtils.close(data, meta); - } else { - IOUtils.closeWhileHandlingException(data, meta); - } - meta = data = null; - } - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormat.java deleted file mode 100644 index d428abf67de72..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormat.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene54; - -import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; - -import java.io.IOException; - -/** - * Lucene 5.4 DocValues format. - * @deprecated Only for reading old 6.0+ segments - */ -@Deprecated -public final class Lucene54DocValuesFormat extends DocValuesFormat { - - /** Sole Constructor */ - public Lucene54DocValuesFormat() { - super("Lucene54"); - } - - @Override - public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - return new Lucene54DocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION); - } - - @Override - public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException { - return new Lucene54DocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION); - } - - static final String DATA_CODEC = "Lucene54DocValuesData"; - static final String DATA_EXTENSION = "dvd"; - static final String META_CODEC = "Lucene54DocValuesMetadata"; - static final String META_EXTENSION = "dvm"; - static final int VERSION_START = 0; - static final int VERSION_CURRENT = VERSION_START; - - // indicates docvalues type - static final byte NUMERIC = 0; - static final byte BINARY = 1; - static final byte SORTED = 2; - static final byte SORTED_SET = 3; - static final byte SORTED_NUMERIC = 4; - - // address terms in blocks of 16 terms - static final int INTERVAL_SHIFT = 4; - static final int INTERVAL_COUNT = 1 << INTERVAL_SHIFT; - static final int INTERVAL_MASK = INTERVAL_COUNT - 1; - - // build reverse index from every 1024th term - static final int REVERSE_INTERVAL_SHIFT = 10; - static final int REVERSE_INTERVAL_COUNT = 1 << REVERSE_INTERVAL_SHIFT; - static final int REVERSE_INTERVAL_MASK = REVERSE_INTERVAL_COUNT - 1; - - // for conversion from reverse index to block - static final int BLOCK_INTERVAL_SHIFT = REVERSE_INTERVAL_SHIFT - INTERVAL_SHIFT; - static final int BLOCK_INTERVAL_COUNT = 1 << BLOCK_INTERVAL_SHIFT; - static final int BLOCK_INTERVAL_MASK = BLOCK_INTERVAL_COUNT - 1; - - /** Compressed using packed blocks of ints. */ - static final int DELTA_COMPRESSED = 0; - /** Compressed by computing the GCD. */ - static final int GCD_COMPRESSED = 1; - /** Compressed by giving IDs to unique values. */ - static final int TABLE_COMPRESSED = 2; - /** Compressed with monotonically increasing values */ - static final int MONOTONIC_COMPRESSED = 3; - /** Compressed with constant value (uses only missing bitset) */ - static final int CONST_COMPRESSED = 4; - /** Compressed with sparse arrays. */ - static final int SPARSE_COMPRESSED = 5; - - /** Uncompressed binary, written directly (fixed length). */ - static final int BINARY_FIXED_UNCOMPRESSED = 0; - /** Uncompressed binary, written directly (variable length). */ - static final int BINARY_VARIABLE_UNCOMPRESSED = 1; - /** Compressed binary with shared prefixes */ - static final int BINARY_PREFIX_COMPRESSED = 2; - - /** Standard storage for sorted set values with 1 level of indirection: - * {@code docId -> address -> ord}. */ - static final int SORTED_WITH_ADDRESSES = 0; - /** Single-valued sorted set values, encoded as sorted values, so no level - * of indirection: {@code docId -> ord}. */ - static final int SORTED_SINGLE_VALUED = 1; - /** Compressed giving IDs to unique sets of values: - * {@code docId -> setId -> ords} */ - static final int SORTED_SET_TABLE = 2; - - /** placeholder for missing offset that means there are no missing values */ - static final int ALL_LIVE = -1; - /** placeholder for missing offset that means all values are missing */ - static final int ALL_MISSING = -2; - - // addressing uses 16k blocks - static final int MONOTONIC_BLOCK_SIZE = 16384; - static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16; -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesProducer.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesProducer.java deleted file mode 100644 index b266cf2fb750b..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesProducer.java +++ /dev/null @@ -1,1847 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene54; - -import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicReader; -import org.apache.lucene.backward_codecs.packed.LegacyDirectReader; -import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BaseTermsEnum; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.ImpactsEnum; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.RandomAccessInput; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Accountables; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LongValues; -import org.apache.lucene.util.PagedBytes; -import org.apache.lucene.util.RamUsageEstimator; -import org.apache.lucene.util.packed.MonotonicBlockPackedReader; -import org.elasticsearch.core.internal.io.IOUtils; -import org.elasticsearch.xpack.lucene.bwc.codecs.index.LegacyBinaryDocValues; -import org.elasticsearch.xpack.lucene.bwc.codecs.index.LegacyBinaryDocValuesWrapper; -import org.elasticsearch.xpack.lucene.bwc.codecs.index.LegacySortedSetDocValues; -import org.elasticsearch.xpack.lucene.bwc.codecs.index.LegacySortedSetDocValuesWrapper; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesConsumer.NumberType.ORDINAL; -import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesConsumer.NumberType.VALUE; - -/** reader for {@link Lucene54DocValuesFormat} */ -final class Lucene54DocValuesProducer extends DocValuesProducer implements Closeable { - private final Map numerics = new HashMap<>(); - private final Map binaries = new HashMap<>(); - private final Map sortedSets = new HashMap<>(); - private final Map sortedNumerics = new HashMap<>(); - private final Map ords = new HashMap<>(); - private final Map ordIndexes = new HashMap<>(); - private final int numFields; - private final AtomicLong ramBytesUsed; - private final IndexInput data; - private final int maxDoc; - - // memory-resident structures - private final Map addressInstances = new HashMap<>(); - private final Map reverseIndexInstances = new HashMap<>(); - private final Map directAddressesMeta = new HashMap<>(); - - private final boolean merging; - - // clone for merge: when merging we don't do any instances.put()s - Lucene54DocValuesProducer(Lucene54DocValuesProducer original) { - assert Thread.holdsLock(original); - numerics.putAll(original.numerics); - binaries.putAll(original.binaries); - sortedSets.putAll(original.sortedSets); - sortedNumerics.putAll(original.sortedNumerics); - ords.putAll(original.ords); - ordIndexes.putAll(original.ordIndexes); - numFields = original.numFields; - ramBytesUsed = new AtomicLong(original.ramBytesUsed.get()); - data = original.data.clone(); - maxDoc = original.maxDoc; - - addressInstances.putAll(original.addressInstances); - reverseIndexInstances.putAll(original.reverseIndexInstances); - merging = true; - } - - /** expert: instantiates a new reader */ - Lucene54DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) - throws IOException { - String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); - this.maxDoc = state.segmentInfo.maxDoc(); - merging = false; - ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass())); - - int version = -1; - int numFields = -1; - - // read in the entries from the metadata file. - try (ChecksumIndexInput in = EndiannessReverserUtil.openChecksumInput(state.directory, metaName, state.context)) { - Throwable priorE = null; - try { - version = CodecUtil.checkIndexHeader( - in, - metaCodec, - Lucene54DocValuesFormat.VERSION_START, - Lucene54DocValuesFormat.VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - numFields = readFields(in, state.fieldInfos); - } catch (Throwable exception) { - priorE = exception; - } finally { - CodecUtil.checkFooter(in, priorE); - } - } - - this.numFields = numFields; - String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); - this.data = EndiannessReverserUtil.openInput(state.directory, dataName, state.context); - boolean success = false; - try { - final int version2 = CodecUtil.checkIndexHeader( - data, - dataCodec, - Lucene54DocValuesFormat.VERSION_START, - Lucene54DocValuesFormat.VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - if (version != version2) { - throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data); - } - - // NOTE: data file is too costly to verify checksum against all the bytes on open, - // but for now we at least verify proper structure of the checksum footer: which looks - // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption - // such as file truncation. - CodecUtil.retrieveChecksum(data); - - success = true; - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(this.data); - } - } - } - - private void readSortedField(FieldInfo info, IndexInput meta) throws IOException { - // sorted = binary + numeric - if (meta.readVInt() != info.number) { - throw new CorruptIndexException("sorted entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.BINARY) { - throw new CorruptIndexException("sorted entry for field: " + info.name + " is corrupt", meta); - } - BinaryEntry b = readBinaryEntry(info, meta); - binaries.put(info.name, b); - - if (meta.readVInt() != info.number) { - throw new CorruptIndexException("sorted entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.NUMERIC) { - throw new CorruptIndexException("sorted entry for field: " + info.name + " is corrupt", meta); - } - NumericEntry n = readNumericEntry(info, meta); - ords.put(info.name, n); - } - - private void readSortedSetFieldWithAddresses(FieldInfo info, IndexInput meta) throws IOException { - // sortedset = binary + numeric (addresses) + ordIndex - if (meta.readVInt() != info.number) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.BINARY) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - BinaryEntry b = readBinaryEntry(info, meta); - binaries.put(info.name, b); - - if (meta.readVInt() != info.number) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.NUMERIC) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - NumericEntry n1 = readNumericEntry(info, meta); - ords.put(info.name, n1); - - if (meta.readVInt() != info.number) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.NUMERIC) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - NumericEntry n2 = readNumericEntry(info, meta); - ordIndexes.put(info.name, n2); - } - - private void readSortedSetFieldWithTable(FieldInfo info, IndexInput meta) throws IOException { - // sortedset table = binary + ordset table + ordset index - if (meta.readVInt() != info.number) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.BINARY) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - - BinaryEntry b = readBinaryEntry(info, meta); - binaries.put(info.name, b); - - if (meta.readVInt() != info.number) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.NUMERIC) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - NumericEntry n = readNumericEntry(info, meta); - ords.put(info.name, n); - } - - private int readFields(IndexInput meta, FieldInfos infos) throws IOException { - int numFields = 0; - int fieldNumber = meta.readVInt(); - while (fieldNumber != -1) { - numFields++; - FieldInfo info = infos.fieldInfo(fieldNumber); - if (info == null) { - // trickier to validate more: because we use multiple entries for "composite" types like sortedset, etc. - throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); - } - byte type = meta.readByte(); - if (type == Lucene54DocValuesFormat.NUMERIC) { - numerics.put(info.name, readNumericEntry(info, meta)); - } else if (type == Lucene54DocValuesFormat.BINARY) { - BinaryEntry b = readBinaryEntry(info, meta); - binaries.put(info.name, b); - } else if (type == Lucene54DocValuesFormat.SORTED) { - readSortedField(info, meta); - } else if (type == Lucene54DocValuesFormat.SORTED_SET) { - SortedSetEntry ss = readSortedSetEntry(meta); - sortedSets.put(info.name, ss); - if (ss.format == Lucene54DocValuesFormat.SORTED_WITH_ADDRESSES) { - readSortedSetFieldWithAddresses(info, meta); - } else if (ss.format == Lucene54DocValuesFormat.SORTED_SET_TABLE) { - readSortedSetFieldWithTable(info, meta); - } else if (ss.format == Lucene54DocValuesFormat.SORTED_SINGLE_VALUED) { - if (meta.readVInt() != fieldNumber) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.SORTED) { - throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt", meta); - } - readSortedField(info, meta); - } else { - throw new AssertionError(); - } - } else if (type == Lucene54DocValuesFormat.SORTED_NUMERIC) { - SortedSetEntry ss = readSortedSetEntry(meta); - sortedNumerics.put(info.name, ss); - if (ss.format == Lucene54DocValuesFormat.SORTED_WITH_ADDRESSES) { - if (meta.readVInt() != fieldNumber) { - throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.NUMERIC) { - throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt", meta); - } - numerics.put(info.name, readNumericEntry(info, meta)); - if (meta.readVInt() != fieldNumber) { - throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.NUMERIC) { - throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt", meta); - } - NumericEntry ordIndex = readNumericEntry(info, meta); - ordIndexes.put(info.name, ordIndex); - } else if (ss.format == Lucene54DocValuesFormat.SORTED_SET_TABLE) { - if (meta.readVInt() != info.number) { - throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.NUMERIC) { - throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt", meta); - } - NumericEntry n = readNumericEntry(info, meta); - ords.put(info.name, n); - } else if (ss.format == Lucene54DocValuesFormat.SORTED_SINGLE_VALUED) { - if (meta.readVInt() != fieldNumber) { - throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt", meta); - } - if (meta.readByte() != Lucene54DocValuesFormat.NUMERIC) { - throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt", meta); - } - numerics.put(info.name, readNumericEntry(info, meta)); - } else { - throw new AssertionError(); - } - } else { - throw new CorruptIndexException("invalid type: " + type, meta); - } - fieldNumber = meta.readVInt(); - } - return numFields; - } - - private NumericEntry readNumericEntry(FieldInfo info, IndexInput meta) throws IOException { - NumericEntry entry = new NumericEntry(); - entry.format = meta.readVInt(); - entry.missingOffset = meta.readLong(); - if (entry.format == Lucene54DocValuesFormat.SPARSE_COMPRESSED) { - // sparse bits need a bit more metadata - entry.numDocsWithValue = meta.readVLong(); - final int blockShift = meta.readVInt(); - entry.monotonicMeta = LegacyDirectMonotonicReader.loadMeta(meta, entry.numDocsWithValue, blockShift); - ramBytesUsed.addAndGet(entry.monotonicMeta.ramBytesUsed()); - directAddressesMeta.put(info.name, entry.monotonicMeta); - } - entry.offset = meta.readLong(); - entry.count = meta.readVLong(); - switch (entry.format) { - case Lucene54DocValuesFormat.CONST_COMPRESSED: - entry.minValue = meta.readLong(); - if (entry.count > Integer.MAX_VALUE) { - // currently just a limitation e.g. of bits interface and so on. - throw new CorruptIndexException("illegal CONST_COMPRESSED count: " + entry.count, meta); - } - break; - case Lucene54DocValuesFormat.GCD_COMPRESSED: - entry.minValue = meta.readLong(); - entry.gcd = meta.readLong(); - entry.bitsPerValue = meta.readVInt(); - break; - case Lucene54DocValuesFormat.TABLE_COMPRESSED: - final int uniqueValues = meta.readVInt(); - if (uniqueValues > 256) { - throw new CorruptIndexException( - "TABLE_COMPRESSED cannot have more than 256 distinct values, got=" + uniqueValues, - meta - ); - } - entry.table = new long[uniqueValues]; - for (int i = 0; i < uniqueValues; ++i) { - entry.table[i] = meta.readLong(); - } - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(entry.table)); - entry.bitsPerValue = meta.readVInt(); - break; - case Lucene54DocValuesFormat.DELTA_COMPRESSED: - entry.minValue = meta.readLong(); - entry.bitsPerValue = meta.readVInt(); - break; - case Lucene54DocValuesFormat.MONOTONIC_COMPRESSED: - final int blockShift = meta.readVInt(); - entry.monotonicMeta = LegacyDirectMonotonicReader.loadMeta(meta, maxDoc + 1, blockShift); - ramBytesUsed.addAndGet(entry.monotonicMeta.ramBytesUsed()); - directAddressesMeta.put(info.name, entry.monotonicMeta); - break; - case Lucene54DocValuesFormat.SPARSE_COMPRESSED: - final byte numberType = meta.readByte(); - switch (numberType) { - case 0: - entry.numberType = VALUE; - break; - case 1: - entry.numberType = ORDINAL; - break; - default: - throw new CorruptIndexException("Number type can only be 0 or 1, got=" + numberType, meta); - } - - // now read the numeric entry for non-missing values - final int fieldNumber = meta.readVInt(); - if (fieldNumber != info.number) { - throw new CorruptIndexException("Field numbers mistmatch: " + fieldNumber + " != " + info.number, meta); - } - final int dvFormat = meta.readByte(); - if (dvFormat != Lucene54DocValuesFormat.NUMERIC) { - throw new CorruptIndexException("Formats mistmatch: " + dvFormat + " != " + Lucene54DocValuesFormat.NUMERIC, meta); - } - entry.nonMissingValues = readNumericEntry(info, meta); - break; - default: - throw new CorruptIndexException("Unknown format: " + entry.format + ", input=", meta); - } - entry.endOffset = meta.readLong(); - return entry; - } - - private BinaryEntry readBinaryEntry(FieldInfo info, IndexInput meta) throws IOException { - BinaryEntry entry = new BinaryEntry(); - entry.format = meta.readVInt(); - entry.missingOffset = meta.readLong(); - entry.minLength = meta.readVInt(); - entry.maxLength = meta.readVInt(); - entry.count = meta.readVLong(); - entry.offset = meta.readLong(); - switch (entry.format) { - case Lucene54DocValuesFormat.BINARY_FIXED_UNCOMPRESSED: - break; - case Lucene54DocValuesFormat.BINARY_PREFIX_COMPRESSED: - entry.addressesOffset = meta.readLong(); - entry.packedIntsVersion = meta.readVInt(); - entry.blockSize = meta.readVInt(); - entry.reverseIndexOffset = meta.readLong(); - break; - case Lucene54DocValuesFormat.BINARY_VARIABLE_UNCOMPRESSED: - entry.addressesOffset = meta.readLong(); - final int blockShift = meta.readVInt(); - entry.addressesMeta = LegacyDirectMonotonicReader.loadMeta(meta, entry.count + 1, blockShift); - ramBytesUsed.addAndGet(entry.addressesMeta.ramBytesUsed()); - directAddressesMeta.put(info.name, entry.addressesMeta); - entry.addressesEndOffset = meta.readLong(); - break; - default: - throw new CorruptIndexException("Unknown format: " + entry.format, meta); - } - return entry; - } - - SortedSetEntry readSortedSetEntry(IndexInput meta) throws IOException { - SortedSetEntry entry = new SortedSetEntry(); - entry.format = meta.readVInt(); - if (entry.format == Lucene54DocValuesFormat.SORTED_SET_TABLE) { - final int totalTableLength = meta.readInt(); - if (totalTableLength > 256) { - throw new CorruptIndexException( - "SORTED_SET_TABLE cannot have more than 256 values in its dictionary, got=" + totalTableLength, - meta - ); - } - entry.table = new long[totalTableLength]; - for (int i = 0; i < totalTableLength; ++i) { - entry.table[i] = meta.readLong(); - } - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(entry.table)); - final int tableSize = meta.readInt(); - if (tableSize > totalTableLength + 1) { // +1 because of the empty set - throw new CorruptIndexException( - "SORTED_SET_TABLE cannot have more set ids than ords in its dictionary, got " - + totalTableLength - + " ords and " - + tableSize - + " sets", - meta - ); - } - entry.tableOffsets = new int[tableSize + 1]; - for (int i = 1; i < entry.tableOffsets.length; ++i) { - entry.tableOffsets[i] = entry.tableOffsets[i - 1] + meta.readInt(); - } - ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(entry.tableOffsets)); - } else if (entry.format != Lucene54DocValuesFormat.SORTED_SINGLE_VALUED - && entry.format != Lucene54DocValuesFormat.SORTED_WITH_ADDRESSES) { - throw new CorruptIndexException("Unknown format: " + entry.format, meta); - } - return entry; - } - - @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { - NumericEntry entry = numerics.get(field.name); - Bits docsWithField; - - if (entry.format == Lucene54DocValuesFormat.SPARSE_COMPRESSED) { - return getSparseNumericDocValues(entry); - } else { - if (entry.missingOffset == Lucene54DocValuesFormat.ALL_MISSING) { - return DocValues.emptyNumeric(); - } else if (entry.missingOffset == Lucene54DocValuesFormat.ALL_LIVE) { - LongValues values = getNumeric(entry); - return new NumericDocValues() { - private int docID = -1; - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - docID++; - if (docID == maxDoc) { - docID = NO_MORE_DOCS; - } - return docID; - } - - @Override - public int advance(int target) { - if (target >= maxDoc) { - docID = NO_MORE_DOCS; - } else { - docID = target; - } - return docID; - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - return true; - } - - @Override - public long cost() { - // TODO - return 0; - } - - @Override - public long longValue() { - return values.get(docID); - } - }; - } else { - docsWithField = getLiveBits(entry.missingOffset, maxDoc); - } - } - final LongValues values = getNumeric(entry); - return new NumericDocValues() { - - int doc = -1; - long value; - - @Override - public long longValue() throws IOException { - return value; - } - - @Override - public int docID() { - return doc; - } - - @Override - public int nextDoc() throws IOException { - return advance(doc + 1); - } - - @Override - public int advance(int target) throws IOException { - for (int doc = target; doc < maxDoc; ++doc) { - value = values.get(doc); - if (value != 0 || docsWithField.get(doc)) { - return this.doc = doc; - } - } - return doc = NO_MORE_DOCS; - } - - @Override - public boolean advanceExact(int target) throws IOException { - doc = target; - value = values.get(doc); - return value != 0 || docsWithField.get(doc); - } - - @Override - public long cost() { - return maxDoc; - } - - }; - } - - @Override - public void checkIntegrity() throws IOException { - CodecUtil.checksumEntireFile(data); - } - - @Override - public String toString() { - return getClass().getSimpleName() + "(fields=" + numFields + ")"; - } - - LongValues getNumeric(NumericEntry entry) throws IOException { - switch (entry.format) { - case Lucene54DocValuesFormat.CONST_COMPRESSED: { - final long constant = entry.minValue; - final Bits live = getLiveBits(entry.missingOffset, (int) entry.count); - return new LongValues() { - @Override - public long get(long index) { - return live.get((int) index) ? constant : 0; - } - }; - } - case Lucene54DocValuesFormat.DELTA_COMPRESSED: { - RandomAccessInput slice = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset); - final long delta = entry.minValue; - final LongValues values = LegacyDirectReader.getInstance(slice, entry.bitsPerValue, 0); - return new LongValues() { - @Override - public long get(long id) { - return delta + values.get(id); - } - }; - } - case Lucene54DocValuesFormat.GCD_COMPRESSED: { - RandomAccessInput slice = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset); - final long min = entry.minValue; - final long mult = entry.gcd; - final LongValues quotientReader = LegacyDirectReader.getInstance(slice, entry.bitsPerValue, 0); - return new LongValues() { - @Override - public long get(long id) { - return min + mult * quotientReader.get(id); - } - }; - } - case Lucene54DocValuesFormat.TABLE_COMPRESSED: { - RandomAccessInput slice = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset); - final long table[] = entry.table; - final LongValues ords = LegacyDirectReader.getInstance(slice, entry.bitsPerValue, 0); - return new LongValues() { - @Override - public long get(long id) { - return table[(int) ords.get(id)]; - } - }; - } - case Lucene54DocValuesFormat.SPARSE_COMPRESSED: - final SparseNumericDocValues values = getSparseNumericDocValues(entry); - final long missingValue; - switch (entry.numberType) { - case ORDINAL: - missingValue = -1L; - break; - case VALUE: - missingValue = 0L; - break; - default: - throw new AssertionError(); - } - return new SparseNumericDocValuesRandomAccessWrapper(values, missingValue); - default: - throw new AssertionError(); - } - } - - static final class SparseNumericDocValues extends NumericDocValues { - - final int docIDsLength; - final LongValues docIds, values; - - int index, doc; - - SparseNumericDocValues(int docIDsLength, LongValues docIDs, LongValues values) { - this.docIDsLength = docIDsLength; - this.docIds = docIDs; - this.values = values; - reset(); - } - - void reset() { - index = -1; - doc = -1; - } - - @Override - public int docID() { - return doc; - } - - @Override - public int nextDoc() throws IOException { - if (index >= docIDsLength - 1) { - index = docIDsLength; - return doc = NO_MORE_DOCS; - } - return doc = (int) docIds.get(++index); - } - - @Override - public int advance(int target) throws IOException { - long loIndex = index; - long step = 1; - long hiIndex; - int hiDoc; - - // gallop forward by exponentially growing the interval - // in order to find an interval so that the target doc - // is in ]lo, hi]. Compared to a regular binary search, - // this optimizes the case that the caller performs many - // advance calls by small deltas - do { - hiIndex = index + step; - if (hiIndex >= docIDsLength) { - hiIndex = docIDsLength; - hiDoc = NO_MORE_DOCS; - break; - } - hiDoc = (int) docIds.get(hiIndex); - if (hiDoc >= target) { - break; - } - step <<= 1; - } while (true); - - // now binary search - while (loIndex + 1 < hiIndex) { - final long midIndex = (loIndex + 1 + hiIndex) >>> 1; - final int midDoc = (int) docIds.get(midIndex); - if (midDoc >= target) { - hiIndex = midIndex; - hiDoc = midDoc; - } else { - loIndex = midIndex; - } - } - - index = (int) hiIndex; - return doc = hiDoc; - } - - @Override - public boolean advanceExact(int target) throws IOException { - if (advance(target) == target) { - return true; - } - --index; - doc = target; - return index >= 0 && docIds.get(index) == target; - } - - @Override - public long longValue() { - assert index >= 0; - assert index < docIDsLength; - return values.get(index); - } - - @Override - public long cost() { - return docIDsLength; - } - } - - static class SparseNumericDocValuesRandomAccessWrapper extends LongValues { - - final SparseNumericDocValues values; - final long missingValue; - - SparseNumericDocValuesRandomAccessWrapper(SparseNumericDocValues values, long missingValue) { - this.values = values; - this.missingValue = missingValue; - } - - @Override - public long get(long longIndex) { - final int index = Math.toIntExact(longIndex); - int doc = values.docID(); - if (doc >= index) { - values.reset(); - } - assert values.docID() < index; - try { - doc = values.advance(index); - } catch (IOException e) { - throw new RuntimeException(e); - } - if (doc == index) { - return values.longValue(); - } else { - return missingValue; - } - } - - } - - LegacyBinaryDocValues getLegacyBinary(FieldInfo field) throws IOException { - BinaryEntry bytes = binaries.get(field.name); - switch (bytes.format) { - case Lucene54DocValuesFormat.BINARY_FIXED_UNCOMPRESSED: - return getFixedBinary(field, bytes); - case Lucene54DocValuesFormat.BINARY_VARIABLE_UNCOMPRESSED: - return getVariableBinary(field, bytes); - case Lucene54DocValuesFormat.BINARY_PREFIX_COMPRESSED: - return getCompressedBinary(field, bytes); - default: - throw new AssertionError(); - } - } - - @Override - public BinaryDocValues getBinary(FieldInfo field) throws IOException { - BinaryEntry be = binaries.get(field.name); - return new LegacyBinaryDocValuesWrapper(getLiveBits(be.missingOffset, maxDoc), getLegacyBinary(field)); - } - - private LegacyBinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { - final IndexInput data = this.data.slice("fixed-binary", bytes.offset, bytes.count * bytes.maxLength); - - final BytesRef term = new BytesRef(bytes.maxLength); - final byte[] buffer = term.bytes; - final int length = term.length = bytes.maxLength; - - return new LongBinaryDocValues() { - @Override - public BytesRef get(long id) { - try { - data.seek(id * length); - data.readBytes(buffer, 0, buffer.length); - return term; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }; - } - - private LegacyBinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { - final RandomAccessInput addressesData = this.data.randomAccessSlice( - bytes.addressesOffset, - bytes.addressesEndOffset - bytes.addressesOffset - ); - final LongValues addresses = LegacyDirectMonotonicReader.getInstance(bytes.addressesMeta, addressesData); - - final IndexInput data = this.data.slice("var-binary", bytes.offset, bytes.addressesOffset - bytes.offset); - final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength)); - final byte buffer[] = term.bytes; - - return new LongBinaryDocValues() { - @Override - public BytesRef get(long id) { - long startAddress = addresses.get(id); - long endAddress = addresses.get(id + 1); - int length = (int) (endAddress - startAddress); - try { - data.seek(startAddress); - data.readBytes(buffer, 0, length); - term.length = length; - return term; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }; - } - - /** returns an address instance for prefix-compressed binary values. */ - private synchronized MonotonicBlockPackedReader getIntervalInstance(FieldInfo field, BinaryEntry bytes) throws IOException { - MonotonicBlockPackedReader addresses = addressInstances.get(field.name); - if (addresses == null) { - data.seek(bytes.addressesOffset); - final long size = (bytes.count + Lucene54DocValuesFormat.INTERVAL_MASK) >>> Lucene54DocValuesFormat.INTERVAL_SHIFT; - addresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size); - if (merging == false) { - addressInstances.put(field.name, addresses); - ramBytesUsed.addAndGet(addresses.ramBytesUsed() + Integer.BYTES); - } - } - return addresses; - } - - /** returns a reverse lookup instance for prefix-compressed binary values. */ - private synchronized ReverseTermsIndex getReverseIndexInstance(FieldInfo field, BinaryEntry bytes) throws IOException { - ReverseTermsIndex index = reverseIndexInstances.get(field.name); - if (index == null) { - index = new ReverseTermsIndex(); - data.seek(bytes.reverseIndexOffset); - long size = (bytes.count + Lucene54DocValuesFormat.REVERSE_INTERVAL_MASK) >>> Lucene54DocValuesFormat.REVERSE_INTERVAL_SHIFT; - index.termAddresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size); - long dataSize = data.readVLong(); - PagedBytes pagedBytes = new PagedBytes(15); - pagedBytes.copy(data, dataSize); - index.terms = pagedBytes.freeze(true); - if (merging == false) { - reverseIndexInstances.put(field.name, index); - ramBytesUsed.addAndGet(index.ramBytesUsed()); - } - } - return index; - } - - private LegacyBinaryDocValues getCompressedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { - final MonotonicBlockPackedReader addresses = getIntervalInstance(field, bytes); - final ReverseTermsIndex index = getReverseIndexInstance(field, bytes); - assert addresses.size() > 0; // we don't have to handle empty case - IndexInput slice = data.slice("terms", bytes.offset, bytes.addressesOffset - bytes.offset); - return new CompressedBinaryDocValues(bytes, addresses, index, slice); - } - - @Override - public SortedDocValues getSorted(FieldInfo field) throws IOException { - final int valueCount = (int) binaries.get(field.name).count; - final LegacyBinaryDocValues binary = getLegacyBinary(field); - NumericEntry entry = ords.get(field.name); - final LongValues ordinals = getNumeric(entry); - if (entry.format == Lucene54DocValuesFormat.SPARSE_COMPRESSED) { - final SparseNumericDocValues sparseValues = ((SparseNumericDocValuesRandomAccessWrapper) ordinals).values; - return new SortedDocValues() { - - @Override - public int ordValue() { - return (int) sparseValues.longValue(); - } - - @Override - public BytesRef lookupOrd(int ord) { - return binary.get(ord); - } - - @Override - public int getValueCount() { - return valueCount; - } - - @Override - public int docID() { - return sparseValues.docID(); - } - - @Override - public int nextDoc() throws IOException { - return sparseValues.nextDoc(); - } - - @Override - public int advance(int target) throws IOException { - return sparseValues.advance(target); - } - - @Override - public boolean advanceExact(int target) throws IOException { - return sparseValues.advanceExact(target); - } - - @Override - public long cost() { - return sparseValues.cost(); - } - - }; - } - return new SortedDocValues() { - private int docID = -1; - private int ord; - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() throws IOException { - assert docID != NO_MORE_DOCS; - while (true) { - docID++; - if (docID == maxDoc) { - docID = NO_MORE_DOCS; - break; - } - ord = (int) ordinals.get(docID); - if (ord != -1) { - break; - } - } - return docID; - } - - @Override - public int advance(int target) throws IOException { - if (target >= maxDoc) { - docID = NO_MORE_DOCS; - return docID; - } else { - docID = target - 1; - return nextDoc(); - } - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - ord = (int) ordinals.get(target); - return ord != -1; - } - - @Override - public int ordValue() { - return ord; - } - - @Override - public long cost() { - // TODO - return 0; - } - - @Override - public BytesRef lookupOrd(int ord) { - return binary.get(ord); - } - - @Override - public int getValueCount() { - return valueCount; - } - - @Override - public int lookupTerm(BytesRef key) throws IOException { - if (binary instanceof CompressedBinaryDocValues) { - return (int) ((CompressedBinaryDocValues) binary).lookupTerm(key); - } else { - return super.lookupTerm(key); - } - } - - @Override - public TermsEnum termsEnum() throws IOException { - if (binary instanceof CompressedBinaryDocValues) { - return ((CompressedBinaryDocValues) binary).getTermsEnum(); - } else { - return super.termsEnum(); - } - } - }; - } - - /** returns an address instance for sortedset ordinal lists */ - private LongValues getOrdIndexInstance(FieldInfo field, NumericEntry entry) throws IOException { - RandomAccessInput data = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset); - return LegacyDirectMonotonicReader.getInstance(entry.monotonicMeta, data); - } - - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - SortedSetEntry ss = sortedNumerics.get(field.name); - if (ss.format == Lucene54DocValuesFormat.SORTED_SINGLE_VALUED) { - NumericEntry numericEntry = numerics.get(field.name); - final LongValues values = getNumeric(numericEntry); - if (numericEntry.format == Lucene54DocValuesFormat.SPARSE_COMPRESSED) { - SparseNumericDocValues sparseValues = ((SparseNumericDocValuesRandomAccessWrapper) values).values; - return new SortedNumericDocValues() { - - @Override - public long nextValue() throws IOException { - return sparseValues.longValue(); - } - - @Override - public int docValueCount() { - return 1; - } - - @Override - public int docID() { - return sparseValues.docID(); - } - - @Override - public int nextDoc() throws IOException { - return sparseValues.nextDoc(); - } - - @Override - public int advance(int target) throws IOException { - return sparseValues.advance(target); - } - - @Override - public boolean advanceExact(int target) throws IOException { - return sparseValues.advanceExact(target); - } - - @Override - public long cost() { - return sparseValues.cost(); - } - - }; - } - final Bits docsWithField = getLiveBits(numericEntry.missingOffset, maxDoc); - return new SortedNumericDocValues() { - int docID = -1; - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - while (true) { - docID++; - if (docID == maxDoc) { - docID = NO_MORE_DOCS; - break; - } - - if (docsWithField.get(docID)) { - // TODO: use .nextSetBit here, at least!! - break; - } - } - return docID; - } - - @Override - public int advance(int target) { - if (target >= maxDoc) { - docID = NO_MORE_DOCS; - return docID; - } else { - docID = target - 1; - return nextDoc(); - } - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - return docsWithField.get(docID); - } - - @Override - public long cost() { - // TODO - return 0; - } - - @Override - public int docValueCount() { - return 1; - } - - @Override - public long nextValue() { - return values.get(docID); - } - }; - } else if (ss.format == Lucene54DocValuesFormat.SORTED_WITH_ADDRESSES) { - NumericEntry numericEntry = numerics.get(field.name); - final LongValues values = getNumeric(numericEntry); - final LongValues ordIndex = getOrdIndexInstance(field, ordIndexes.get(field.name)); - - return new SortedNumericDocValues() { - long startOffset; - long endOffset; - int docID = -1; - long upto; - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - while (true) { - docID++; - if (docID == maxDoc) { - docID = NO_MORE_DOCS; - return docID; - } - startOffset = ordIndex.get(docID); - endOffset = ordIndex.get(docID + 1L); - if (endOffset > startOffset) { - break; - } - } - upto = startOffset; - return docID; - } - - @Override - public int advance(int target) { - if (target >= maxDoc) { - docID = NO_MORE_DOCS; - return docID; - } else { - docID = target - 1; - return nextDoc(); - } - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - startOffset = ordIndex.get(docID); - endOffset = ordIndex.get(docID + 1L); - upto = startOffset; - return endOffset > startOffset; - } - - @Override - public long cost() { - // TODO - return 0; - } - - @Override - public int docValueCount() { - return (int) (endOffset - startOffset); - } - - @Override - public long nextValue() { - return values.get(upto++); - } - }; - } else if (ss.format == Lucene54DocValuesFormat.SORTED_SET_TABLE) { - NumericEntry entry = ords.get(field.name); - final LongValues ordinals = getNumeric(entry); - - final long[] table = ss.table; - final int[] offsets = ss.tableOffsets; - return new SortedNumericDocValues() { - int startOffset; - int endOffset; - int docID = -1; - int upto; - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - while (true) { - docID++; - if (docID == maxDoc) { - docID = NO_MORE_DOCS; - return docID; - } - int ord = (int) ordinals.get(docID); - startOffset = offsets[ord]; - endOffset = offsets[ord + 1]; - if (endOffset > startOffset) { - break; - } - } - upto = startOffset; - return docID; - } - - @Override - public int advance(int target) { - if (target >= maxDoc) { - docID = NO_MORE_DOCS; - return docID; - } else { - docID = target - 1; - return nextDoc(); - } - } - - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - int ord = (int) ordinals.get(docID); - startOffset = offsets[ord]; - endOffset = offsets[ord + 1]; - upto = startOffset; - return endOffset > startOffset; - } - - @Override - public long cost() { - // TODO - return 0; - } - - @Override - public int docValueCount() { - return endOffset - startOffset; - } - - @Override - public long nextValue() { - return table[upto++]; - } - }; - } else { - throw new AssertionError(); - } - } - - @Override - public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { - SortedSetEntry ss = sortedSets.get(field.name); - switch (ss.format) { - case Lucene54DocValuesFormat.SORTED_SINGLE_VALUED: - return DocValues.singleton(getSorted(field)); - case Lucene54DocValuesFormat.SORTED_WITH_ADDRESSES: - return getSortedSetWithAddresses(field); - case Lucene54DocValuesFormat.SORTED_SET_TABLE: - return getSortedSetTable(field, ss); - default: - throw new AssertionError(); - } - } - - private SortedSetDocValues getSortedSetWithAddresses(FieldInfo field) throws IOException { - final long valueCount = binaries.get(field.name).count; - // we keep the byte[]s and list of ords on disk, these could be large - final LongBinaryDocValues binary = (LongBinaryDocValues) getLegacyBinary(field); - final LongValues ordinals = getNumeric(ords.get(field.name)); - // but the addresses to the ord stream are in RAM - final LongValues ordIndex = getOrdIndexInstance(field, ordIndexes.get(field.name)); - - return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() { - long startOffset; - long offset; - long endOffset; - - @Override - public long nextOrd() { - if (offset == endOffset) { - return NO_MORE_ORDS; - } else { - long ord = ordinals.get(offset); - offset++; - return ord; - } - } - - @Override - public void setDocument(int docID) { - startOffset = offset = ordIndex.get(docID); - endOffset = ordIndex.get(docID + 1L); - } - - @Override - public BytesRef lookupOrd(long ord) { - return binary.get(ord); - } - - @Override - public long getValueCount() { - return valueCount; - } - - @Override - public long lookupTerm(BytesRef key) { - if (binary instanceof CompressedBinaryDocValues) { - return ((CompressedBinaryDocValues) binary).lookupTerm(key); - } else { - return super.lookupTerm(key); - } - } - - @Override - public TermsEnum termsEnum() throws IOException { - if (binary instanceof CompressedBinaryDocValues) { - return ((CompressedBinaryDocValues) binary).getTermsEnum(); - } else { - return super.termsEnum(); - } - } - }, maxDoc); - } - - private SortedSetDocValues getSortedSetTable(FieldInfo field, SortedSetEntry ss) throws IOException { - final long valueCount = binaries.get(field.name).count; - final LongBinaryDocValues binary = (LongBinaryDocValues) getLegacyBinary(field); - final NumericEntry ordinalsEntry = ords.get(field.name); - final LongValues ordinals = getNumeric(ordinalsEntry); - - final long[] table = ss.table; - final int[] offsets = ss.tableOffsets; - - return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() { - - int offset, startOffset, endOffset; - - @Override - public void setDocument(int docID) { - final int ord = (int) ordinals.get(docID); - offset = startOffset = offsets[ord]; - endOffset = offsets[ord + 1]; - } - - @Override - public long nextOrd() { - if (offset == endOffset) { - return NO_MORE_ORDS; - } else { - return table[offset++]; - } - } - - @Override - public BytesRef lookupOrd(long ord) { - return binary.get(ord); - } - - @Override - public long getValueCount() { - return valueCount; - } - - @Override - public long lookupTerm(BytesRef key) { - if (binary instanceof CompressedBinaryDocValues) { - return ((CompressedBinaryDocValues) binary).lookupTerm(key); - } else { - return super.lookupTerm(key); - } - } - - @Override - public TermsEnum termsEnum() throws IOException { - if (binary instanceof CompressedBinaryDocValues) { - return ((CompressedBinaryDocValues) binary).getTermsEnum(); - } else { - return super.termsEnum(); - } - } - }, maxDoc); - } - - private Bits getLiveBits(final long offset, final int count) throws IOException { - if (offset == Lucene54DocValuesFormat.ALL_MISSING) { - return new Bits.MatchNoBits(count); - } else if (offset == Lucene54DocValuesFormat.ALL_LIVE) { - return new Bits.MatchAllBits(count); - } else { - int length = (int) ((count + 7L) >>> 3); - final RandomAccessInput in = data.randomAccessSlice(offset, length); - return new Bits() { - @Override - public boolean get(int index) { - try { - return (in.readByte(index >> 3) & (1 << (index & 7))) != 0; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public int length() { - return count; - } - }; - } - } - - private SparseNumericDocValues getSparseNumericDocValues(NumericEntry entry) throws IOException { - final RandomAccessInput docIdsData = this.data.randomAccessSlice(entry.missingOffset, entry.offset - entry.missingOffset); - final LongValues docIDs = LegacyDirectMonotonicReader.getInstance(entry.monotonicMeta, docIdsData); - final LongValues values = getNumeric(entry.nonMissingValues); // cannot be sparse - return new SparseNumericDocValues(Math.toIntExact(entry.numDocsWithValue), docIDs, values); - } - - @Override - public synchronized DocValuesProducer getMergeInstance() { - return new Lucene54DocValuesProducer(this); - } - - @Override - public void close() throws IOException { - data.close(); - } - - /** metadata entry for a numeric docvalues field */ - static class NumericEntry { - private NumericEntry() {} - - /** offset to the bitset representing docsWithField, or -1 if no documents have missing values */ - long missingOffset; - /** offset to the actual numeric values */ - public long offset; - /** end offset to the actual numeric values */ - public long endOffset; - /** bits per value used to pack the numeric values */ - public int bitsPerValue; - - int format; - /** count of values written */ - public long count; - - /** monotonic meta */ - public LegacyDirectMonotonicReader.Meta monotonicMeta; - - long minValue; - long gcd; - long table[]; - - /** for sparse compression */ - long numDocsWithValue; - NumericEntry nonMissingValues; - Lucene54DocValuesConsumer.NumberType numberType; - } - - /** metadata entry for a binary docvalues field */ - static class BinaryEntry { - private BinaryEntry() {} - - /** offset to the bitset representing docsWithField, or -1 if no documents have missing values */ - long missingOffset; - /** offset to the actual binary values */ - long offset; - - int format; - /** count of values written */ - public long count; - int minLength; - int maxLength; - /** offset to the addressing data that maps a value to its slice of the byte[] */ - public long addressesOffset, addressesEndOffset; - /** meta data for addresses */ - public LegacyDirectMonotonicReader.Meta addressesMeta; - /** offset to the reverse index */ - public long reverseIndexOffset; - /** packed ints version used to encode addressing information */ - public int packedIntsVersion; - /** packed ints blocksize */ - public int blockSize; - } - - /** metadata entry for a sorted-set docvalues field */ - static class SortedSetEntry { - private SortedSetEntry() {} - - int format; - - long[] table; - int[] tableOffsets; - } - - // internally we compose complex dv (sorted/sortedset) from other ones - abstract static class LongBinaryDocValues extends LegacyBinaryDocValues { - @Override - public final BytesRef get(int docID) { - return get((long) docID); - } - - abstract BytesRef get(long id); - } - - // used for reverse lookup to a small range of blocks - static class ReverseTermsIndex implements Accountable { - public MonotonicBlockPackedReader termAddresses; - public PagedBytes.Reader terms; - - @Override - public long ramBytesUsed() { - return termAddresses.ramBytesUsed() + terms.ramBytesUsed(); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("term bytes", terms)); - resources.add(Accountables.namedAccountable("term addresses", termAddresses)); - return Collections.unmodifiableList(resources); - } - - @Override - public String toString() { - return getClass().getSimpleName() + "(size=" + termAddresses.size() + ")"; - } - } - - // in the compressed case, we add a few additional operations for - // more efficient reverse lookup and enumeration - static final class CompressedBinaryDocValues extends LongBinaryDocValues { - final long numValues; - final long numIndexValues; - final int maxTermLength; - final MonotonicBlockPackedReader addresses; - final IndexInput data; - final CompressedBinaryTermsEnum termsEnum; - final PagedBytes.Reader reverseTerms; - final MonotonicBlockPackedReader reverseAddresses; - final long numReverseIndexValues; - - CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, ReverseTermsIndex index, IndexInput data) - throws IOException { - this.maxTermLength = bytes.maxLength; - this.numValues = bytes.count; - this.addresses = addresses; - this.numIndexValues = addresses.size(); - this.data = data; - this.reverseTerms = index.terms; - this.reverseAddresses = index.termAddresses; - this.numReverseIndexValues = reverseAddresses.size(); - this.termsEnum = getTermsEnum(data); - } - - @Override - public BytesRef get(long id) { - try { - termsEnum.seekExact(id); - return termsEnum.term(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - long lookupTerm(BytesRef key) { - try { - switch (termsEnum.seekCeil(key)) { - case FOUND: - return termsEnum.ord(); - case NOT_FOUND: - return -termsEnum.ord() - 1; - default: - return -numValues - 1; - } - } catch (IOException bogus) { - throw new RuntimeException(bogus); - } - } - - TermsEnum getTermsEnum() throws IOException { - return getTermsEnum(data.clone()); - } - - private CompressedBinaryTermsEnum getTermsEnum(IndexInput input) throws IOException { - return new CompressedBinaryTermsEnum(input); - } - - class CompressedBinaryTermsEnum extends BaseTermsEnum { - private long currentOrd = -1; - // offset to the start of the current block - private long currentBlockStart; - private final IndexInput input; - // delta from currentBlockStart to start of each term - private final int offsets[] = new int[Lucene54DocValuesFormat.INTERVAL_COUNT]; - private final byte buffer[] = new byte[2 * Lucene54DocValuesFormat.INTERVAL_COUNT - 1]; - - private final BytesRef term = new BytesRef(maxTermLength); - private final BytesRef firstTerm = new BytesRef(maxTermLength); - private final BytesRef scratch = new BytesRef(); - - CompressedBinaryTermsEnum(IndexInput input) throws IOException { - this.input = input; - input.seek(0); - } - - private void readHeader() throws IOException { - firstTerm.length = input.readVInt(); - input.readBytes(firstTerm.bytes, 0, firstTerm.length); - input.readBytes(buffer, 0, Lucene54DocValuesFormat.INTERVAL_COUNT - 1); - if (buffer[0] == -1) { - readShortAddresses(); - } else { - readByteAddresses(); - } - currentBlockStart = input.getFilePointer(); - } - - // read single byte addresses: each is delta - 2 - // (shared prefix byte and length > 0 are both implicit) - private void readByteAddresses() throws IOException { - int addr = 0; - for (int i = 1; i < offsets.length; i++) { - addr += 2 + (buffer[i - 1] & 0xFF); - offsets[i] = addr; - } - } - - // read double byte addresses: each is delta - 2 - // (shared prefix byte and length > 0 are both implicit) - private void readShortAddresses() throws IOException { - input.readBytes(buffer, Lucene54DocValuesFormat.INTERVAL_COUNT - 1, Lucene54DocValuesFormat.INTERVAL_COUNT); - int addr = 0; - for (int i = 1; i < offsets.length; i++) { - int x = i << 1; - addr += 2 + ((buffer[x - 1] << 8) | (buffer[x] & 0xFF)); - offsets[i] = addr; - } - } - - // set term to the first term - private void readFirstTerm() throws IOException { - term.length = firstTerm.length; - System.arraycopy(firstTerm.bytes, firstTerm.offset, term.bytes, 0, term.length); - } - - // read term at offset, delta encoded from first term - private void readTerm(int offset) throws IOException { - int start = input.readByte() & 0xFF; - System.arraycopy(firstTerm.bytes, firstTerm.offset, term.bytes, 0, start); - int suffix = offsets[offset] - offsets[offset - 1] - 1; - input.readBytes(term.bytes, start, suffix); - term.length = start + suffix; - } - - @Override - public BytesRef next() throws IOException { - currentOrd++; - if (currentOrd >= numValues) { - return null; - } else { - int offset = (int) (currentOrd & Lucene54DocValuesFormat.INTERVAL_MASK); - if (offset == 0) { - // switch to next block - readHeader(); - readFirstTerm(); - } else { - readTerm(offset); - } - return term; - } - } - - // binary search reverse index to find smaller - // range of blocks to search - long binarySearchIndex(BytesRef text) throws IOException { - long low = 0; - long high = numReverseIndexValues - 1; - while (low <= high) { - long mid = (low + high) >>> 1; - reverseTerms.fill(scratch, reverseAddresses.get(mid)); - int cmp = scratch.compareTo(text); - - if (cmp < 0) { - low = mid + 1; - } else if (cmp > 0) { - high = mid - 1; - } else { - return mid; - } - } - return high; - } - - // binary search against first term in block range - // to find term's block - long binarySearchBlock(BytesRef text, long low, long high) throws IOException { - while (low <= high) { - long mid = (low + high) >>> 1; - input.seek(addresses.get(mid)); - term.length = input.readVInt(); - input.readBytes(term.bytes, 0, term.length); - int cmp = term.compareTo(text); - - if (cmp < 0) { - low = mid + 1; - } else if (cmp > 0) { - high = mid - 1; - } else { - return mid; - } - } - return high; - } - - @Override - public SeekStatus seekCeil(BytesRef text) throws IOException { - // locate block: narrow to block range with index, then search blocks - final long block; - long indexPos = binarySearchIndex(text); - if (indexPos < 0) { - block = 0; - } else { - long low = indexPos << Lucene54DocValuesFormat.BLOCK_INTERVAL_SHIFT; - long high = Math.min(numIndexValues - 1, low + Lucene54DocValuesFormat.BLOCK_INTERVAL_MASK); - block = Math.max(low, binarySearchBlock(text, low, high)); - } - - // position before block, then scan to term. - input.seek(addresses.get(block)); - currentOrd = (block << Lucene54DocValuesFormat.INTERVAL_SHIFT) - 1; - - while (next() != null) { - int cmp = term.compareTo(text); - if (cmp == 0) { - return SeekStatus.FOUND; - } else if (cmp > 0) { - return SeekStatus.NOT_FOUND; - } - } - return SeekStatus.END; - } - - @Override - public void seekExact(long ord) throws IOException { - long block = ord >>> Lucene54DocValuesFormat.INTERVAL_SHIFT; - if (block != currentOrd >>> Lucene54DocValuesFormat.INTERVAL_SHIFT) { - // switch to different block - input.seek(addresses.get(block)); - readHeader(); - } - - currentOrd = ord; - - int offset = (int) (ord & Lucene54DocValuesFormat.INTERVAL_MASK); - if (offset == 0) { - readFirstTerm(); - } else { - input.seek(currentBlockStart + offsets[offset - 1]); - readTerm(offset); - } - } - - @Override - public BytesRef term() throws IOException { - return term; - } - - @Override - public long ord() throws IOException { - return currentOrd; - } - - @Override - public int docFreq() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public long totalTermFreq() throws IOException { - return -1; - } - - @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public ImpactsEnum impacts(int flags) throws IOException { - throw new UnsupportedOperationException(); - } - - } - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java deleted file mode 100644 index 43a24574297c3..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene60; - -import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; -import org.apache.lucene.codecs.CompoundFormat; -import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; -import org.elasticsearch.xpack.lucene.bwc.codecs.lucene50.Lucene50SegmentInfoFormat; -import org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat; - -import java.util.Objects; - -/** - * Implements the Lucene 6.0 index format. - * - * @deprecated Only for 6.0 back compat - */ -@Deprecated -public class Lucene60Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene50SegmentInfoFormat()); - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); - private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); - private final StoredFieldsFormat storedFieldsFormat; - private final DocValuesFormat defaultDocValuesFormat = new Lucene54DocValuesFormat(); - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDocValuesFormat; - } - }; - - /** - * Instantiates a new codec. - */ - public Lucene60Codec() { - this(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - } - - /** - * Instantiates a new codec, specifying the stored fields compression - * mode to use. - * @param mode stored fields compression mode to use for newly - * flushed/merged segments. - */ - public Lucene60Codec(Lucene50StoredFieldsFormat.Mode mode) { - super("Lucene60"); - this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode)); - } - - @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - - @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; - } - - @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; - } - - @Override - public final LiveDocsFormat liveDocsFormat() { - return liveDocsFormat; - } - - @Override - public final CompoundFormat compoundFormat() { - return compoundFormat; - } - - @Override - public DocValuesFormat docValuesFormat() { - return docValuesFormat; - } - -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java deleted file mode 100644 index 2f805a4881744..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene62; - -import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; -import org.apache.lucene.codecs.CompoundFormat; -import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; -import org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat; - -import java.util.Objects; - -/** - * Implements the Lucene 6.2 index format. - * - * @deprecated Only for 6.2 back compat - */ -@Deprecated -public class Lucene62Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene62SegmentInfoFormat()); - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); - private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); - private final StoredFieldsFormat storedFieldsFormat; - private final DocValuesFormat defaultDocValuesFormat = new Lucene54DocValuesFormat(); - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDocValuesFormat; - } - }; - - public Lucene62Codec() { - this(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - } - - public Lucene62Codec(Lucene50StoredFieldsFormat.Mode mode) { - super("Lucene62"); - this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode)); - } - - @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - - @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; - } - - @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; - } - - @Override - public final LiveDocsFormat liveDocsFormat() { - return liveDocsFormat; - } - - @Override - public final CompoundFormat compoundFormat() { - return compoundFormat; - } - - @Override - public DocValuesFormat docValuesFormat() { - return docValuesFormat; - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java deleted file mode 100644 index b700c39591819..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene62; - -import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedNumericSelector; -import org.apache.lucene.search.SortedNumericSortField; -import org.apache.lucene.search.SortedSetSelector; -import org.apache.lucene.search.SortedSetSortField; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.Version; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; - -/** - * Lucene 6.2 Segment info format. - * @deprecated Only for reading old 6.2+ segments - */ -@Deprecated -public class Lucene62SegmentInfoFormat extends SegmentInfoFormat { - - public Lucene62SegmentInfoFormat() {} - - @Override - public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException { - final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene62SegmentInfoFormat.SI_EXTENSION); - try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(dir, fileName, context)) { - Throwable priorE = null; - SegmentInfo si = null; - try { - int format = CodecUtil.checkIndexHeader( - input, - Lucene62SegmentInfoFormat.CODEC_NAME, - Lucene62SegmentInfoFormat.VERSION_START, - Lucene62SegmentInfoFormat.VERSION_CURRENT, - segmentID, - "" - ); - final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt()); - - final int docCount = input.readInt(); - if (docCount < 0) { - throw new CorruptIndexException("invalid docCount: " + docCount, input); - } - final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; - - final Map diagnostics = input.readMapOfStrings(); - final Set files = input.readSetOfStrings(); - final Map attributes = input.readMapOfStrings(); - - int numSortFields = input.readVInt(); - Sort indexSort; - if (numSortFields > 0) { - SortField[] sortFields = new SortField[numSortFields]; - for (int i = 0; i < numSortFields; i++) { - String fieldName = input.readString(); - int sortTypeID = input.readVInt(); - SortField.Type sortType; - SortedSetSelector.Type sortedSetSelector = null; - SortedNumericSelector.Type sortedNumericSelector = null; - switch (sortTypeID) { - case 0 -> sortType = SortField.Type.STRING; - case 1 -> sortType = SortField.Type.LONG; - case 2 -> sortType = SortField.Type.INT; - case 3 -> sortType = SortField.Type.DOUBLE; - case 4 -> sortType = SortField.Type.FLOAT; - case 5 -> { - sortType = SortField.Type.STRING; - byte selector = input.readByte(); - if (selector == 0) { - sortedSetSelector = SortedSetSelector.Type.MIN; - } else if (selector == 1) { - sortedSetSelector = SortedSetSelector.Type.MAX; - } else if (selector == 2) { - sortedSetSelector = SortedSetSelector.Type.MIDDLE_MIN; - } else if (selector == 3) { - sortedSetSelector = SortedSetSelector.Type.MIDDLE_MAX; - } else { - throw new CorruptIndexException("invalid index SortedSetSelector ID: " + selector, input); - } - } - case 6 -> { - byte type = input.readByte(); - if (type == 0) { - sortType = SortField.Type.LONG; - } else if (type == 1) { - sortType = SortField.Type.INT; - } else if (type == 2) { - sortType = SortField.Type.DOUBLE; - } else if (type == 3) { - sortType = SortField.Type.FLOAT; - } else { - throw new CorruptIndexException("invalid index SortedNumericSortField type ID: " + type, input); - } - byte numericSelector = input.readByte(); - if (numericSelector == 0) { - sortedNumericSelector = SortedNumericSelector.Type.MIN; - } else if (numericSelector == 1) { - sortedNumericSelector = SortedNumericSelector.Type.MAX; - } else { - throw new CorruptIndexException("invalid index SortedNumericSelector ID: " + numericSelector, input); - } - } - default -> throw new CorruptIndexException("invalid index sort field type ID: " + sortTypeID, input); - } - byte b = input.readByte(); - boolean reverse; - if (b == 0) { - reverse = true; - } else if (b == 1) { - reverse = false; - } else { - throw new CorruptIndexException("invalid index sort reverse: " + b, input); - } - - if (sortedSetSelector != null) { - sortFields[i] = new SortedSetSortField(fieldName, reverse, sortedSetSelector); - } else if (sortedNumericSelector != null) { - sortFields[i] = new SortedNumericSortField(fieldName, sortType, reverse, sortedNumericSelector); - } else { - sortFields[i] = new SortField(fieldName, sortType, reverse); - } - - Object missingValue; - b = input.readByte(); - if (b == 0) { - missingValue = null; - } else { - switch (sortType) { - case STRING: - if (b == 1) { - missingValue = SortField.STRING_LAST; - } else if (b == 2) { - missingValue = SortField.STRING_FIRST; - } else { - throw new CorruptIndexException("invalid missing value flag: " + b, input); - } - break; - case LONG: - if (b != 1) { - throw new CorruptIndexException("invalid missing value flag: " + b, input); - } - missingValue = input.readLong(); - break; - case INT: - if (b != 1) { - throw new CorruptIndexException("invalid missing value flag: " + b, input); - } - missingValue = input.readInt(); - break; - case DOUBLE: - if (b != 1) { - throw new CorruptIndexException("invalid missing value flag: " + b, input); - } - missingValue = Double.longBitsToDouble(input.readLong()); - break; - case FLOAT: - if (b != 1) { - throw new CorruptIndexException("invalid missing value flag: " + b, input); - } - missingValue = Float.intBitsToFloat(input.readInt()); - break; - default: - throw new AssertionError("unhandled sortType=" + sortType); - } - } - if (missingValue != null) { - sortFields[i].setMissingValue(missingValue); - } - } - indexSort = new Sort(sortFields); - } else if (numSortFields < 0) { - throw new CorruptIndexException("invalid index sort field count: " + numSortFields, input); - } else { - indexSort = null; - } - - si = new SegmentInfo( - dir, - version, - null, - segment, - docCount, - isCompoundFile, - null, - diagnostics, - segmentID, - attributes, - indexSort - ); - si.setFiles(files); - } catch (Throwable exception) { - priorE = exception; - } finally { - CodecUtil.checkFooter(input, priorE); - } - return si; - } - } - - @Override - public void write(Directory dir, SegmentInfo info, IOContext ioContext) { - throw new UnsupportedOperationException("This format can only be used for reading"); - } - - /** File extension used to store {@link SegmentInfo}. */ - public static final String SI_EXTENSION = "si"; - static final String CODEC_NAME = "Lucene62SegmentInfo"; - static final int VERSION_START = 0; - static final int VERSION_MULTI_VALUED_SORT = 1; - static final int VERSION_CURRENT = VERSION_MULTI_VALUED_SORT; -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java deleted file mode 100644 index bc9fa098476c1..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - * - * Modifications copyright (C) 2021 Elasticsearch B.V. - */ - -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene70; - -import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; -import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; -import org.apache.lucene.codecs.CompoundFormat; -import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; - -public class BWCLucene70Codec extends BWCCodec { - - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); - private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); - private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); - private final StoredFieldsFormat storedFieldsFormat; - private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene70"); - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { - @Override - public DocValuesFormat getDocValuesFormatForField(String field) { - return defaultDVFormat; - } - }; - - public BWCLucene70Codec() { - super("BWCLucene70Codec"); - storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - } - - @Override - public FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; - } - - @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; - } - - @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - - @Override - public LiveDocsFormat liveDocsFormat() { - return liveDocsFormat; - } - - @Override - public CompoundFormat compoundFormat() { - return compoundFormat; - } - - @Override - public final DocValuesFormat docValuesFormat() { - return docValuesFormat; - } -} diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec deleted file mode 100644 index 6e5205d664f2d..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0; you may not use this file except in compliance with the Elastic License -# 2.0. -# - -org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec -org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec -org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60Codec diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat deleted file mode 100644 index 2d46b4bca3d0c..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormatTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormatTests.java deleted file mode 100644 index 23d76046fa168..0000000000000 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormatTests.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.lucene.bwc.codecs.lucene54; - -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase; -import org.apache.lucene.tests.util.TestUtil; - -public class Lucene54DocValuesFormatTests extends BaseDocValuesFormatTestCase { - - private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene54DocValuesFormat()); - - @Override - protected Codec getCodec() { - return codec; - } -} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 7dddd8969bd2d..d7dbdcbc0d9e8 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -277,7 +277,6 @@ public class Constants { "cluster:monitor/xpack/info", "cluster:monitor/xpack/info/aggregate_metric", "cluster:monitor/xpack/info/analytics", - "cluster:monitor/xpack/info/archive", "cluster:monitor/xpack/info/ccr", "cluster:monitor/xpack/info/data_streams", "cluster:monitor/xpack/info/data_tiers", @@ -333,7 +332,6 @@ public class Constants { "cluster:monitor/xpack/usage", "cluster:monitor/xpack/usage/aggregate_metric", "cluster:monitor/xpack/usage/analytics", - "cluster:monitor/xpack/usage/archive", "cluster:monitor/xpack/usage/ccr", "cluster:monitor/xpack/usage/data_streams", "cluster:monitor/xpack/usage/data_tiers", diff --git a/x-pack/qa/repository-old-versions/build.gradle b/x-pack/qa/repository-old-versions/build.gradle deleted file mode 100644 index fc2e96ec66045..0000000000000 --- a/x-pack/qa/repository-old-versions/build.gradle +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.Architecture -import org.elasticsearch.gradle.OS -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.internal.info.BuildParams -import org.elasticsearch.gradle.internal.test.AntFixture -import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -import org.elasticsearch.gradle.transform.UnzipTransform -import org.gradle.api.internal.artifacts.ArtifactAttributes - -apply plugin: 'elasticsearch.jdk-download' -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.rest-resources' - -configurations { - oldesFixture -} - -dependencies { - oldesFixture project(':test:fixtures:old-elasticsearch') - testImplementation project(':client:rest-high-level') -} - -jdks { - legacy { - vendor = 'adoptium' - version = '8u302+b08' - platform = OS.current().name().toLowerCase() - architecture = Architecture.current().name().toLowerCase() - } -} - -restResources { - restApi { - include '_common', 'search' - } - restTests { - includeCore 'search/390_doc_values_search.yml' - } -} - -if (Os.isFamily(Os.FAMILY_WINDOWS)) { - logger.warn("Disabling repository-old-versions tests because we can't get the pid file on windows") - tasks.named("testingConventions").configure { enabled = false } -} else { - /* Register a gradle artifact transformation to unpack resolved elasticsearch distributions. We only resolve - * zip files here. Using artifact transforms allow a better caching of the downloaded distros as the - * transformed (unpacked) distro will be cached by gradle resulting in less unpacking - * - * To avoid testing against too many old versions, always pick first and last version per major - */ - project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { - transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE); - transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); - }); - - for (String versionString : ['5.0.0', '5.6.16', '6.0.0', '6.8.20']) { - Version version = Version.fromString(versionString) - String packageName = 'org.elasticsearch.distribution.zip' - String artifact = "${packageName}:elasticsearch:${version}@zip" - String versionNoDots = version.toString().replace('.', '_') - String configName = "es${versionNoDots}" - - def config = configurations.create(configName) - config.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); - dependencies.add(configName, artifact) - - String repoLocation = "${buildDir}/cluster/shared/repo/${versionNoDots}" - String clusterName = versionNoDots - - def testClusterProvider = testClusters.register(clusterName) { - testDistribution = 'DEFAULT' - numberOfNodes = 2 - versions = [project.version, project.version] // to test full cluster restart - - setting 'path.repo', repoLocation - setting 'xpack.license.self_generated.type', 'trial' - - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - - setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' - setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' - } - - TaskProvider fixture = tasks.register("oldES${versionNoDots}Fixture", AntFixture) { - dependsOn project.configurations.oldesFixture, jdks.legacy, config - executable = "${BuildParams.runtimeJavaHome}/bin/java" - env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}" - // old versions of Elasticsearch need JAVA_HOME - env 'JAVA_HOME', jdks.legacy.javaHomePath - // If we are running on certain arm systems we need to explicitly set the stack size to overcome JDK page size bug - if (Architecture.current() == Architecture.AARCH64) { - env 'ES_JAVA_OPTS', '-Xss512k' - } - def dataPath = "${baseDir}/data" - args 'oldes.OldElasticsearch', - baseDir, - "${ -> config.getSingleFile().toPath()}", - false, - "path.repo: ${repoLocation}", - "path.data: ${dataPath}" - if (version.onOrAfter('6.8.0') && Architecture.current() == Architecture.AARCH64) { - // We need to explicitly disable ML when running old ES versions on ARM - args 'xpack.ml.enabled: false' - } - doFirst { - delete(dataPath) - mkdir(dataPath) - } - maxWaitInSeconds 60 - waitCondition = { fixture, ant -> - // the fixture writes the ports file when Elasticsearch's HTTP service - // is ready, so we can just wait for the file to exist - return fixture.portsFile.exists() - } - } - - tasks.register("javaRestTestBeforeRestart#${versionNoDots}", StandaloneRestIntegTestTask) { - useCluster testClusterProvider - dependsOn fixture - doFirst { - delete(repoLocation) - mkdir(repoLocation) - } - systemProperty 'tests.after_restart', 'false' - } - - tasks.register("javaRestTestAfterRestart#${versionNoDots}", StandaloneRestIntegTestTask) { - useCluster testClusterProvider - dependsOn fixture - dependsOn "javaRestTestBeforeRestart#${versionNoDots}" - systemProperty 'tests.after_restart', 'true' - - doFirst { - testClusterProvider.get().goToNextVersion() - } - } - - tasks.matching { it.name.startsWith("javaRestTest") && it.name.endsWith(versionNoDots) }.configureEach { - it.systemProperty "tests.repo.location", repoLocation - it.systemProperty "tests.es.version", version.toString() - - /* Use a closure on the string to delay evaluation until right before we - * run the integration tests so that we can be sure that the file is - * ready. */ - it.nonInputProperties.systemProperty "tests.es.port", "${-> fixture.get().addressAndPort}" - it.nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusterProvider.get().allHttpSocketURI.join(",")}") - it.nonInputProperties.systemProperty('tests.clustername', "${-> testClusterProvider.get().getName()}") - } - - tasks.named("check").configure { - dependsOn "javaRestTestAfterRestart#${versionNoDots}" - } - } -} - diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java deleted file mode 100644 index 6868c1f5ea3a5..0000000000000 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos; - -import com.carrotsearch.randomizedtesting.RandomizedTest; -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.apache.http.HttpHost; -import org.elasticsearch.Version; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.junit.Before; - -import java.io.IOException; - -/** - * Tests doc-value-based searches against indices imported from clusters older than N-1. - * We reuse the YAML tests in search/390_doc_values_search.yml but have to do the setup - * manually here as the setup is done on the old cluster for which we have to use the - * low-level REST client instead of the YAML set up that only knows how to talk to - * newer ES versions. - * - * We mimic the setup in search/390_doc_values_search.yml here, but adapt it to work - * against older version clusters. - */ -public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase { - - final Version oldVersion = Version.fromString(System.getProperty("tests.es.version")); - static boolean setupDone; - - public DocValueOnlyFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @ParametersFactory - public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(); - } - - @Override - protected boolean preserveClusterUponCompletion() { - return true; - } - - @Override - protected Settings restClientSettings() { - String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - - @Override - protected boolean skipSetupSections() { - // setup in the YAML file is replaced by the method below - return true; - } - - @Before - public void setupIndex() throws IOException { - final boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); - if (afterRestart) { - return; - } - - // The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only - // initialized later, so we do it when running the first test - if (setupDone) { - return; - } - - setupDone = true; - - String repoLocation = PathUtils.get(System.getProperty("tests.repo.location")) - .resolve(RandomizedTest.getContext().getTargetClass().getName()) - .toString(); - - String indexName = "test"; - String repoName = "doc_values_repo"; - String snapshotName = "snap"; - String[] basicTypes = new String[] { - "byte", - "double", - "float", - "half_float", - "integer", - "long", - "short", - "boolean", - "keyword", - "ip", - "geo_point" }; // date is manually added as it need further configuration - - int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); - try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { - Request createIndex = new Request("PUT", "/" + indexName); - int numberOfShards = randomIntBetween(1, 3); - - boolean multiTypes = oldVersion.before(Version.V_7_0_0); - - XContentBuilder settingsBuilder = XContentFactory.jsonBuilder() - .startObject() - .startObject("settings") - .field("index.number_of_shards", numberOfShards) - .endObject() - .startObject("mappings"); - if (multiTypes) { - settingsBuilder.startObject("doc"); - } - settingsBuilder.field("dynamic", false).startObject("properties"); - for (String type : basicTypes) { - settingsBuilder.startObject(type).field("type", type).endObject(); - } - settingsBuilder.startObject("date").field("type", "date").field("format", "yyyy/MM/dd").endObject(); - if (multiTypes) { - settingsBuilder.endObject(); - } - settingsBuilder.endObject().endObject().endObject(); - - createIndex.setJsonEntity(Strings.toString(settingsBuilder)); - assertOK(oldEs.performRequest(createIndex)); - - Request doc1 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "1"); - doc1.addParameter("refresh", "true"); - XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder() - .startObject() - .field("byte", 1) - .field("double", 1.0) - .field("float", 1.0) - .field("half_float", 1.0) - .field("integer", 1) - .field("long", 1) - .field("short", 1) - .field("date", "2017/01/01") - .field("keyword", "key1") - .field("boolean", false) - .field("ip", "192.168.0.1") - .array("geo_point", 13.5, 34.89) - .endObject(); - doc1.setJsonEntity(Strings.toString(bodyDoc1)); - assertOK(oldEs.performRequest(doc1)); - - Request doc2 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "2"); - doc2.addParameter("refresh", "true"); - XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder() - .startObject() - .field("byte", 2) - .field("double", 2.0) - .field("float", 2.0) - .field("half_float", 2.0) - .field("integer", 2) - .field("long", 2) - .field("short", 2) - .field("date", "2017/01/02") - .field("keyword", "key2") - .field("boolean", true) - .field("ip", "192.168.0.2") - .array("geo_point", -63.24, 31.0) - .endObject(); - doc2.setJsonEntity(Strings.toString(bodyDoc2)); - assertOK(oldEs.performRequest(doc2)); - - // register repo on old ES and take snapshot - Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest.setJsonEntity(""" - {"type":"fs","settings":{"location":"%s"}} - """.formatted(repoLocation)); - assertOK(oldEs.performRequest(createRepoRequest)); - - Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); - createSnapshotRequest.addParameter("wait_for_completion", "true"); - createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); - assertOK(oldEs.performRequest(createSnapshotRequest)); - } - - // register repo on new ES and restore snapshot - Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest2.setJsonEntity(""" - {"type":"fs","settings":{"location":"%s"}} - """.formatted(repoLocation)); - assertOK(client().performRequest(createRepoRequest2)); - - final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore"); - createRestoreRequest.addParameter("wait_for_completion", "true"); - createRestoreRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); - assertOK(client().performRequest(createRestoreRequest)); - - // add mappings (they will be auto-converted later) - Request putMappingsRequest = new Request("PUT", "/" + indexName + "/_mappings"); - XContentBuilder mappingsBuilder = XContentFactory.jsonBuilder().startObject().startObject("properties"); - for (String type : basicTypes) { - mappingsBuilder.startObject(type).field("type", type).endObject(); - } - mappingsBuilder.startObject("date").field("type", "date").field("format", "yyyy/MM/dd").endObject(); - mappingsBuilder.endObject().endObject(); - putMappingsRequest.setJsonEntity(Strings.toString(mappingsBuilder)); - assertOK(client().performRequest(putMappingsRequest)); - } -} diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java deleted file mode 100644 index 2ce6e3ee7a806..0000000000000 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ /dev/null @@ -1,499 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos; - -import org.apache.http.HttpHost; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; -import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; -import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; -import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.CloseIndexRequest; -import org.elasticsearch.client.indices.GetMappingsRequest; -import org.elasticsearch.client.indices.PutMappingRequest; -import org.elasticsearch.cluster.SnapshotsInProgress; -import org.elasticsearch.cluster.metadata.MappingMetadata; -import org.elasticsearch.cluster.routing.Murmur3HashFunction; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.snapshots.SnapshotInfo; -import org.elasticsearch.snapshots.SnapshotState; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.ObjectPath; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.json.JsonXContent; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.startsWith; - -public class OldRepositoryAccessIT extends ESRestTestCase { - - @Override - protected boolean preserveClusterUponCompletion() { - return true; - } - - @Override - protected Settings restClientSettings() { - String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - - @SuppressWarnings("removal") - protected static RestHighLevelClient highLevelClient(RestClient client) { - return new RestHighLevelClient(client, ignore -> {}, Collections.emptyList()) { - }; - } - - public void testOldRepoAccess() throws IOException { - runTest(false); - } - - public void testOldSourceOnlyRepoAccess() throws IOException { - runTest(true); - } - - @SuppressWarnings("removal") - public void runTest(boolean sourceOnlyRepository) throws IOException { - boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart")); - String repoLocation = System.getProperty("tests.repo.location"); - repoLocation = PathUtils.get(repoLocation).resolve("source_only_" + sourceOnlyRepository).toString(); - Version oldVersion = Version.fromString(System.getProperty("tests.es.version")); - assumeTrue( - "source only repositories only supported since ES 6.5.0", - sourceOnlyRepository == false || oldVersion.onOrAfter(Version.fromString("6.5.0")) - ); - - int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port")); - String indexName; - if (sourceOnlyRepository) { - indexName = "source_only_test_index"; - } else { - indexName = "test_index"; - } - int numDocs = 10; - int extraDocs = 1; - final Set expectedIds = new HashSet<>(); - try ( - RestHighLevelClient client = highLevelClient(adminClient()); - RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build() - ) { - if (afterRestart == false) { - beforeRestart(sourceOnlyRepository, repoLocation, oldVersion, numDocs, extraDocs, expectedIds, client, oldEs, indexName); - } else { - afterRestart(indexName); - } - } - } - - private void afterRestart(String indexName) throws IOException { - ensureGreen("restored_" + indexName); - ensureGreen("mounted_full_copy_" + indexName); - ensureGreen("mounted_shared_cache_" + indexName); - } - - @SuppressWarnings("removal") - private void beforeRestart( - boolean sourceOnlyRepository, - String repoLocation, - Version oldVersion, - int numDocs, - int extraDocs, - Set expectedIds, - RestHighLevelClient client, - RestClient oldEs, - String indexName - ) throws IOException { - String repoName = "repo_" + indexName; - String snapshotName = "snap_" + indexName; - Request createIndex = new Request("PUT", "/" + indexName); - int numberOfShards = randomIntBetween(1, 3); - - XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject().startObject("settings"); - settingsBuilder.field("index.number_of_shards", numberOfShards); - - // 6.5.0 started using soft-deletes, but it was only enabled by default on 7.0 - if (oldVersion.onOrAfter(Version.fromString("6.5.0")) && oldVersion.before(Version.fromString("7.0.0")) && randomBoolean()) { - settingsBuilder.field("index.soft_deletes.enabled", true); - } - - settingsBuilder.endObject().endObject(); - - createIndex.setJsonEntity(Strings.toString(settingsBuilder)); - assertOK(oldEs.performRequest(createIndex)); - - for (int i = 0; i < numDocs + extraDocs; i++) { - String id = "testdoc" + i; - expectedIds.add(id); - // use multiple types for ES versions < 6.0.0 - String type = getType(oldVersion, id); - Request doc = new Request("PUT", "/" + indexName + "/" + type + "/" + id); - doc.addParameter("refresh", "true"); - doc.setJsonEntity(sourceForDoc(i)); - assertOK(oldEs.performRequest(doc)); - } - - for (int i = 0; i < extraDocs; i++) { - String id = randomFrom(expectedIds); - expectedIds.remove(id); - String type = getType(oldVersion, id); - Request doc = new Request("DELETE", "/" + indexName + "/" + type + "/" + id); - doc.addParameter("refresh", "true"); - oldEs.performRequest(doc); - } - - // register repo on old ES and take snapshot - Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName); - createRepoRequest.setJsonEntity(sourceOnlyRepository ? """ - {"type":"source","settings":{"location":"%s","delegate_type":"fs"}} - """.formatted(repoLocation) : """ - {"type":"fs","settings":{"location":"%s"}} - """.formatted(repoLocation)); - assertOK(oldEs.performRequest(createRepoRequest)); - - Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName); - createSnapshotRequest.addParameter("wait_for_completion", "true"); - createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}"); - assertOK(oldEs.performRequest(createSnapshotRequest)); - - // register repo on new ES - Settings.Builder repoSettingsBuilder = Settings.builder().put("location", repoLocation); - if (sourceOnlyRepository) { - repoSettingsBuilder.put("delegate_type", "fs"); - } - ElasticsearchAssertions.assertAcked( - client.snapshot() - .createRepository( - new PutRepositoryRequest(repoName).type(sourceOnlyRepository ? "source" : "fs").settings(repoSettingsBuilder), - RequestOptions.DEFAULT - ) - ); - - // list snapshots on new ES - List snapshotInfos = client.snapshot() - .get(new GetSnapshotsRequest(repoName).snapshots(new String[] { "_all" }), RequestOptions.DEFAULT) - .getSnapshots(); - assertThat(snapshotInfos, hasSize(1)); - SnapshotInfo snapshotInfo = snapshotInfos.get(0); - assertEquals(snapshotName, snapshotInfo.snapshotId().getName()); - assertEquals(repoName, snapshotInfo.repository()); - assertEquals(Arrays.asList(indexName), snapshotInfo.indices()); - assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); - assertEquals(numberOfShards, snapshotInfo.successfulShards()); - assertEquals(numberOfShards, snapshotInfo.totalShards()); - assertEquals(0, snapshotInfo.failedShards()); - assertEquals(oldVersion, snapshotInfo.version()); - - // list specific snapshot on new ES - snapshotInfos = client.snapshot() - .get(new GetSnapshotsRequest(repoName).snapshots(new String[] { snapshotName }), RequestOptions.DEFAULT) - .getSnapshots(); - assertThat(snapshotInfos, hasSize(1)); - snapshotInfo = snapshotInfos.get(0); - assertEquals(snapshotName, snapshotInfo.snapshotId().getName()); - assertEquals(repoName, snapshotInfo.repository()); - assertEquals(Arrays.asList(indexName), snapshotInfo.indices()); - assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); - assertEquals(numberOfShards, snapshotInfo.successfulShards()); - assertEquals(numberOfShards, snapshotInfo.totalShards()); - assertEquals(0, snapshotInfo.failedShards()); - assertEquals(oldVersion, snapshotInfo.version()); - - // list advanced snapshot info on new ES - SnapshotsStatusResponse snapshotsStatusResponse = client.snapshot() - .status(new SnapshotsStatusRequest(repoName).snapshots(new String[] { snapshotName }), RequestOptions.DEFAULT); - assertThat(snapshotsStatusResponse.getSnapshots(), hasSize(1)); - SnapshotStatus snapshotStatus = snapshotsStatusResponse.getSnapshots().get(0); - assertEquals(snapshotName, snapshotStatus.getSnapshot().getSnapshotId().getName()); - assertEquals(repoName, snapshotStatus.getSnapshot().getRepository()); - assertEquals(Sets.newHashSet(indexName), snapshotStatus.getIndices().keySet()); - assertEquals(SnapshotsInProgress.State.SUCCESS, snapshotStatus.getState()); - assertEquals(numberOfShards, snapshotStatus.getShardsStats().getDoneShards()); - assertEquals(numberOfShards, snapshotStatus.getShardsStats().getTotalShards()); - assertEquals(0, snapshotStatus.getShardsStats().getFailedShards()); - assertThat(snapshotStatus.getStats().getTotalSize(), greaterThan(0L)); - assertThat(snapshotStatus.getStats().getTotalFileCount(), greaterThan(0)); - - // restore / mount and check whether searches work - restoreMountAndVerify( - numDocs, - expectedIds, - client, - numberOfShards, - sourceOnlyRepository, - oldVersion, - indexName, - repoName, - snapshotName - ); - - // close indices - assertTrue(client.indices().close(new CloseIndexRequest("restored_" + indexName), RequestOptions.DEFAULT).isShardsAcknowledged()); - assertTrue( - client.indices().close(new CloseIndexRequest("mounted_full_copy_" + indexName), RequestOptions.DEFAULT).isShardsAcknowledged() - ); - assertTrue( - client.indices() - .close(new CloseIndexRequest("mounted_shared_cache_" + indexName), RequestOptions.DEFAULT) - .isShardsAcknowledged() - ); - - // restore / mount again - restoreMountAndVerify( - numDocs, - expectedIds, - client, - numberOfShards, - sourceOnlyRepository, - oldVersion, - indexName, - repoName, - snapshotName - ); - } - - private String getType(Version oldVersion, String id) { - return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Math.abs(Murmur3HashFunction.hash(id) % 2) : 0); - } - - private static String sourceForDoc(int i) { - return "{\"test\":\"test" + i + "\",\"val\":" + i + "}"; - } - - @SuppressWarnings("removal") - private void restoreMountAndVerify( - int numDocs, - Set expectedIds, - RestHighLevelClient client, - int numberOfShards, - boolean sourceOnlyRepository, - Version oldVersion, - String indexName, - String repoName, - String snapshotName - ) throws IOException { - // restore index - RestoreSnapshotResponse restoreSnapshotResponse = client.snapshot() - .restore( - new RestoreSnapshotRequest(repoName, snapshotName).indices(indexName) - .renamePattern("(.+)") - .renameReplacement("restored_$1") - .waitForCompletion(true), - RequestOptions.DEFAULT - ); - assertNotNull(restoreSnapshotResponse.getRestoreInfo()); - assertEquals(numberOfShards, restoreSnapshotResponse.getRestoreInfo().totalShards()); - assertEquals(numberOfShards, restoreSnapshotResponse.getRestoreInfo().successfulShards()); - - ensureGreen("restored_" + indexName); - - MappingMetadata mapping = client.indices() - .getMapping(new GetMappingsRequest().indices("restored_" + indexName), RequestOptions.DEFAULT) - .mappings() - .get("restored_" + indexName); - logger.info("mapping for {}: {}", mapping.type(), mapping.source().string()); - Map root = mapping.sourceAsMap(); - assertThat(root, hasKey("_meta")); - assertThat(root.get("_meta"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map meta = (Map) root.get("_meta"); - assertThat(meta, hasKey("legacy_mappings")); - assertThat(meta.get("legacy_mappings"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map legacyMappings = (Map) meta.get("legacy_mappings"); - assertThat(legacyMappings.keySet(), not(empty())); - for (Map.Entry entry : legacyMappings.entrySet()) { - String type = entry.getKey(); - assertThat(type, startsWith("doc")); - assertThat(entry.getValue(), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map legacyMapping = (Map) entry.getValue(); - assertThat(legacyMapping, hasKey("properties")); - assertThat(legacyMapping.get("properties"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map propertiesMapping = (Map) legacyMapping.get("properties"); - assertThat(propertiesMapping, hasKey("val")); - assertThat(propertiesMapping.get("val"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map valMapping = (Map) propertiesMapping.get("val"); - assertThat(valMapping, hasKey("type")); - assertEquals("long", valMapping.get("type")); - } - - // run a search against the index - assertDocs("restored_" + indexName, numDocs, expectedIds, client, sourceOnlyRepository, oldVersion); - - // mount as full copy searchable snapshot - Request mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); - mountRequest.setJsonEntity( - "{\"index\": \"" - + indexName - + "\",\"renamed_index\": \"mounted_full_copy_" - + indexName - + "\",\"index_settings\": {\"index.number_of_replicas\": 1}}" - ); - mountRequest.addParameter("wait_for_completion", "true"); - ObjectPath mountResponse = ObjectPath.createFromResponse(client().performRequest(mountRequest)); - assertNotNull(mountResponse.evaluate("snapshot")); - assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.total")); - assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.successful")); - - ensureGreen("mounted_full_copy_" + indexName); - - // run a search against the index - assertDocs("mounted_full_copy_" + indexName, numDocs, expectedIds, client, sourceOnlyRepository, oldVersion); - - // mount as shared cache searchable snapshot - mountRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_mount"); - mountRequest.setJsonEntity("{\"index\": \"" + indexName + "\",\"renamed_index\": \"mounted_shared_cache_" + indexName + "\"}"); - mountRequest.addParameter("wait_for_completion", "true"); - mountRequest.addParameter("storage", "shared_cache"); - mountResponse = ObjectPath.createFromResponse(client().performRequest(mountRequest)); - assertNotNull(mountResponse.evaluate("snapshot")); - assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.total")); - assertEquals(numberOfShards, (int) mountResponse.evaluate("snapshot.shards.successful")); - - // run a search against the index - assertDocs("mounted_shared_cache_" + indexName, numDocs, expectedIds, client, sourceOnlyRepository, oldVersion); - } - - @SuppressWarnings("removal") - private void assertDocs( - String index, - int numDocs, - Set expectedIds, - RestHighLevelClient client, - boolean sourceOnlyRepository, - Version oldVersion - ) throws IOException { - RequestOptions v7RequestOptions = RequestOptions.DEFAULT.toBuilder() - .addHeader("Content-Type", "application/vnd.elasticsearch+json;compatible-with=7") - .addHeader("Accept", "application/vnd.elasticsearch+json;compatible-with=7") - .build(); - RequestOptions randomRequestOptions = randomBoolean() ? RequestOptions.DEFAULT : v7RequestOptions; - - // run a search against the index - SearchResponse searchResponse = client.search(new SearchRequest(index), randomRequestOptions); - logger.info(searchResponse); - // check hit count - assertEquals(numDocs, searchResponse.getHits().getTotalHits().value); - // check that _index is properly set - assertTrue(Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getIndex).allMatch(index::equals)); - // check that all _ids are there - assertEquals(expectedIds, Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toSet())); - // check that _source is present - assertTrue(Arrays.stream(searchResponse.getHits().getHits()).allMatch(SearchHit::hasSource)); - // check that correct _source present for each document - for (SearchHit h : searchResponse.getHits().getHits()) { - assertEquals(sourceForDoc(getIdAsNumeric(h.getId())), h.getSourceAsString()); - } - - String id = randomFrom(expectedIds); - int num = getIdAsNumeric(id); - // run a search using runtime fields against the index - searchResponse = client.search( - new SearchRequest(index).source( - SearchSourceBuilder.searchSource() - .query(QueryBuilders.matchQuery("val", num)) - .runtimeMappings(Map.of("val", Map.of("type", "long"))) - ), - randomRequestOptions - ); - logger.info(searchResponse); - assertEquals(1, searchResponse.getHits().getTotalHits().value); - assertEquals(id, searchResponse.getHits().getHits()[0].getId()); - assertEquals(sourceForDoc(num), searchResponse.getHits().getHits()[0].getSourceAsString()); - - if (sourceOnlyRepository == false) { - // check that doc values can be accessed by (reverse) sorting on numeric val field - // first add mapping for field (this will be done automatically in the future) - XContentBuilder mappingBuilder = JsonXContent.contentBuilder(); - mappingBuilder.startObject().startObject("properties"); - mappingBuilder.startObject("val").field("type", "long").endObject(); - mappingBuilder.endObject().endObject(); - assertTrue( - client.indices().putMapping(new PutMappingRequest(index).source(mappingBuilder), RequestOptions.DEFAULT).isAcknowledged() - ); - - // search using reverse sort on val - searchResponse = client.search( - new SearchRequest(index).source( - SearchSourceBuilder.searchSource() - .query(QueryBuilders.matchAllQuery()) - .sort(SortBuilders.fieldSort("val").order(SortOrder.DESC)) - ), - randomRequestOptions - ); - logger.info(searchResponse); - // check sort order - assertEquals( - expectedIds.stream().sorted(Comparator.comparingInt(this::getIdAsNumeric).reversed()).collect(Collectors.toList()), - Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()) - ); - - if (oldVersion.before(Version.fromString("6.0.0"))) { - // search on _type and check that results contain _type information - String randomType = getType(oldVersion, randomFrom(expectedIds)); - long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count(); - searchResponse = client.search( - new SearchRequest(index).source(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType))), - randomRequestOptions - ); - logger.info(searchResponse); - assertEquals(typeCount, searchResponse.getHits().getTotalHits().value); - for (SearchHit hit : searchResponse.getHits().getHits()) { - DocumentField typeField = hit.field("_type"); - assertNotNull(typeField); - assertThat(typeField.getValue(), instanceOf(String.class)); - assertEquals(randomType, typeField.getValue()); - } - } - } - } - - private int getIdAsNumeric(String id) { - return Integer.parseInt(id.substring("testdoc".length())); - } -}