Update archival indices logic to support ES 7 indices

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java

@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.lucene.bwc.codecs;
 
+import org.apache.lucene.backward_codecs.lucene86.Lucene86Codec;
+import org.apache.lucene.backward_codecs.lucene87.Lucene87Codec;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FieldsConsumer;
@@ -26,6 +28,8 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -118,7 +122,14 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) {
     }
 
     public static SegmentInfo wrap(SegmentInfo segmentInfo) {
-        final Codec codec = segmentInfo.getCodec();
+        // special handling for Lucene87Codec (which is currently bundled with Lucene)
+        // Use BWCLucene87Codec instead as that one extends BWCCodec (similar to all other older codecs)
+        Codec codec = segmentInfo.getCodec();
+        if (codec instanceof Lucene86Codec) {
+            codec = new BWCLucene86Codec();
+        } else if (codec instanceof Lucene87Codec) {
+            codec = new BWCLucene87Codec();
+        }
         final SegmentInfo segmentInfo1 = new SegmentInfo(
             segmentInfo.dir,
             // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation)

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java

@@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86;
+
+import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;
+import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;
+import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
+import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat;
+import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
+
+import java.util.Objects;
+
+public class BWCLucene86Codec extends BWCCodec {
+
+    private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
+    private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat());
+    private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat());
+    private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+    private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+    private final PointsFormat pointsFormat = new Lucene86PointsFormat();
+    private final PostingsFormat defaultFormat;
+
+    private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
+        @Override
+        public PostingsFormat getPostingsFormatForField(String field) {
+            return BWCLucene86Codec.this.getPostingsFormatForField(field);
+        }
+    };
+
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return BWCLucene86Codec.this.getDocValuesFormatForField(field);
+        }
+    };
+
+    private final StoredFieldsFormat storedFieldsFormat;
+
+    /** Instantiates a new codec. */
+    public BWCLucene86Codec() {
+        super("BWCLucene86Codec");
+        this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(Lucene50StoredFieldsFormat.Mode.BEST_SPEED));
+        this.defaultFormat = new Lucene84PostingsFormat();
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public TermVectorsFormat termVectorsFormat() {
+        return vectorsFormat;
+    }
+
+    @Override
+    public PostingsFormat postingsFormat() {
+        return postingsFormat;
+    }
+
+    @Override
+    public final FieldInfosFormat fieldInfosFormat() {
+        return fieldInfosFormat;
+    }
+
+    @Override
+    public SegmentInfoFormat segmentInfoFormat() {
+        return segmentInfosFormat;
+    }
+
+    @Override
+    public final LiveDocsFormat liveDocsFormat() {
+        return liveDocsFormat;
+    }
+
+    @Override
+    public CompoundFormat compoundFormat() {
+        return compoundFormat;
+    }
+
+    @Override
+    public PointsFormat pointsFormat() {
+        return pointsFormat;
+    }
+
+    @Override
+    public final KnnVectorsFormat knnVectorsFormat() {
+        return KnnVectorsFormat.EMPTY;
+    }
+
+    /**
+     * Returns the postings format that should be used for writing new segments of <code>field</code>.
+     *
+     * <p>The default implementation always returns "Lucene84".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public PostingsFormat getPostingsFormatForField(String field) {
+        return defaultFormat;
+    }
+
+    /**
+     * Returns the docvalues format that should be used for writing new segments of <code>field</code>
+     * .
+     *
+     * <p>The default implementation always returns "Lucene80".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public DocValuesFormat getDocValuesFormatForField(String field) {
+        return defaultDVFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+
+    private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80");
+
+    private final NormsFormat normsFormat = new Lucene80NormsFormat();
+
+    @Override
+    public NormsFormat normsFormat() {
+        return normsFormat;
+    }
+
+}

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java

@@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene87;
+
+import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;
+import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
+import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;
+import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
+import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat;
+import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat;
+import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
+
+public class BWCLucene87Codec extends BWCCodec {
+
+    private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
+    private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat());
+    private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat());
+    private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+    private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+    private final PointsFormat pointsFormat = new Lucene86PointsFormat();
+    private final PostingsFormat defaultFormat;
+
+    private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
+        @Override
+        public PostingsFormat getPostingsFormatForField(String field) {
+            return BWCLucene87Codec.this.getPostingsFormatForField(field);
+        }
+    };
+
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return BWCLucene87Codec.this.getDocValuesFormatForField(field);
+        }
+    };
+
+    private final StoredFieldsFormat storedFieldsFormat;
+
+    /** Instantiates a new codec. */
+    public BWCLucene87Codec() {
+        super("BWCLucene87Codec");
+        this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION);
+        this.defaultFormat = new Lucene84PostingsFormat();
+        this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION);
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public TermVectorsFormat termVectorsFormat() {
+        return vectorsFormat;
+    }
+
+    @Override
+    public PostingsFormat postingsFormat() {
+        return postingsFormat;
+    }
+
+    @Override
+    public final FieldInfosFormat fieldInfosFormat() {
+        return fieldInfosFormat;
+    }
+
+    @Override
+    public SegmentInfoFormat segmentInfoFormat() {
+        return segmentInfosFormat;
+    }
+
+    @Override
+    public final LiveDocsFormat liveDocsFormat() {
+        return liveDocsFormat;
+    }
+
+    @Override
+    public CompoundFormat compoundFormat() {
+        return compoundFormat;
+    }
+
+    @Override
+    public PointsFormat pointsFormat() {
+        return pointsFormat;
+    }
+
+    @Override
+    public final KnnVectorsFormat knnVectorsFormat() {
+        return KnnVectorsFormat.EMPTY;
+    }
+
+    /**
+     * Returns the postings format that should be used for writing new segments of <code>field</code>.
+     *
+     * <p>The default implementation always returns "Lucene84".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public PostingsFormat getPostingsFormatForField(String field) {
+        return defaultFormat;
+    }
+
+    /**
+     * Returns the docvalues format that should be used for writing new segments of <code>field</code>
+     * .
+     *
+     * <p>The default implementation always returns "Lucene80".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public DocValuesFormat getDocValuesFormatForField(String field) {
+        return defaultDVFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+
+    private final DocValuesFormat defaultDVFormat;
+
+    private final NormsFormat normsFormat = new Lucene80NormsFormat();
+
+    @Override
+    public NormsFormat normsFormat() {
+        return normsFormat;
+    }
+
+}

x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec

@@ -5,6 +5,8 @@
 # 2.0.
 #
 
+org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec
+org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70Codec
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec

x-pack/qa/repository-old-versions-7x/build.gradle

@@ -0,0 +1,12 @@
+import org.elasticsearch.gradle.internal.test.RestIntegTestTask
+import org.elasticsearch.gradle.Version
+
+apply plugin: 'elasticsearch.internal-java-rest-test'
+
+tasks.named("javaRestTest").configure {
+  def versionString = "7.17.25"
+  // def versionString = "7.9.0"
+  systemProperty 'tests.old_cluster_version', versionString
+  usesDefaultDistribution()
+  usesBwcDistribution(Version.fromString(versionString))
+}