Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update archival indices logic to support ES 7 indices #116565

Open
wants to merge 31 commits into
base: main
Choose a base branch
from
Open
Changes from 25 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
95ba9c4
WIP
cbuescher Nov 6, 2024
d2817fa
Trying to add new qa test project for 7x archival indices
cbuescher Nov 21, 2024
1bd696d
iter
cbuescher Nov 22, 2024
85b61cd
Add basic search test
cbuescher Nov 22, 2024
93b95fb
Rework OldMappingIT
cbuescher Nov 25, 2024
6b8177c
iter
cbuescher Nov 25, 2024
b27ab02
Merge branch 'main' into add-bwcLucene87Codec
cbuescher Nov 25, 2024
33f3f36
Use system property for version
cbuescher Nov 25, 2024
dce64b2
Add BWCLucene86Codec
cbuescher Nov 25, 2024
8aae226
Merge branch 'main' into add-bwcLucene87Codec
cbuescher Nov 25, 2024
2f3b6d0
Fix codec name
cbuescher Nov 25, 2024
feaee28
Change version to 7.9.0
cbuescher Nov 26, 2024
becea6b
Merge branch 'main' into add-bwcLucene87Codec
cbuescher Nov 26, 2024
41d69c7
Add modified version of OldRepositoryAccessIT
cbuescher Nov 26, 2024
8b3acb8
Adding back test for source_only repo
cbuescher Nov 26, 2024
55ed6d0
Add looping over versions
cbuescher Nov 26, 2024
1351e2e
Change looping over version
cbuescher Nov 27, 2024
4d35161
Fix cluster version checks
cbuescher Nov 27, 2024
d4be0ac
No snapshot cache for old cluster
cbuescher Nov 27, 2024
9919159
Add DocValueOnlyFieldsIT yaml rest test
cbuescher Nov 27, 2024
2406751
Fix wiring of yaml specs to test tasks
breskeby Nov 27, 2024
85d01c2
Make DocValueOnlyFieldsIT work for V7x
cbuescher Nov 27, 2024
f17527f
Using 7.9.0 instead of 7.16
cbuescher Nov 27, 2024
6cabe33
Merge branch 'main' into add-bwcLucene87Codec
cbuescher Nov 27, 2024
da3db30
Cleanups
cbuescher Nov 27, 2024
4a46d80
Merge branch 'main' into add-bwcLucene87Codec
cbuescher Nov 28, 2024
01ba741
Add _field_names disabling to new tests
cbuescher Nov 28, 2024
3d257dd
Merge branch 'main' into add-bwcLucene87Codec
cbuescher Nov 29, 2024
c7693b3
Add restart cluster test
cbuescher Nov 29, 2024
3bf231d
Merge branch 'main' into add-bwcLucene87Codec
cbuescher Dec 2, 2024
050d195
Pulling in test changes from 117649
cbuescher Dec 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -118,6 +118,7 @@
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;
@@ -408,6 +409,10 @@ protected static boolean has(ProductFeature feature) {
}

protected List<HttpHost> parseClusterHosts(String hostsString) {
return parseClusterHosts(hostsString, this::buildHttpHost);
}

public static List<HttpHost> parseClusterHosts(String hostsString, BiFunction<String, Integer, HttpHost> httpHostSupplier) {
String[] stringUrls = hostsString.split(",");
List<HttpHost> hosts = new ArrayList<>(stringUrls.length);
for (String stringUrl : stringUrls) {
@@ -417,7 +422,7 @@ protected List<HttpHost> parseClusterHosts(String hostsString) {
}
String host = stringUrl.substring(0, portSeparator);
int port = Integer.valueOf(stringUrl.substring(portSeparator + 1));
hosts.add(buildHttpHost(host, port));
hosts.add(httpHostSupplier.apply(host, port));
}
return unmodifiableList(hosts);
}
Original file line number Diff line number Diff line change
@@ -7,6 +7,8 @@

package org.elasticsearch.xpack.lucene.bwc.codecs;

import org.apache.lucene.backward_codecs.lucene86.Lucene86Codec;
import org.apache.lucene.backward_codecs.lucene87.Lucene87Codec;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldsConsumer;
@@ -26,6 +28,8 @@
import org.apache.lucene.index.Terms;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec;
import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec;

import java.io.IOException;
import java.util.ArrayList;
@@ -118,7 +122,14 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) {
}

public static SegmentInfo wrap(SegmentInfo segmentInfo) {
final Codec codec = segmentInfo.getCodec();
// special handling for Lucene87Codec (which is currently bundled with Lucene)
// Use BWCLucene87Codec instead as that one extends BWCCodec (similar to all other older codecs)
Codec codec = segmentInfo.getCodec();
if (codec instanceof Lucene86Codec) {
codec = new BWCLucene86Codec();
} else if (codec instanceof Lucene87Codec) {
codec = new BWCLucene87Codec();
}
final SegmentInfo segmentInfo1 = new SegmentInfo(
segmentInfo.dir,
// Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation)
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86;

import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;
import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;
import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat;
import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat;
import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;

import java.util.Objects;

public class BWCLucene86Codec extends BWCCodec {

private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat());
private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat());
private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
private final PointsFormat pointsFormat = new Lucene86PointsFormat();
private final PostingsFormat defaultFormat;

private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return BWCLucene86Codec.this.getPostingsFormatForField(field);
}
};

private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
return BWCLucene86Codec.this.getDocValuesFormatForField(field);
}
};

private final StoredFieldsFormat storedFieldsFormat;

/** Instantiates a new codec. */
public BWCLucene86Codec() {
super("BWCLucene86Codec");
this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(Lucene50StoredFieldsFormat.Mode.BEST_SPEED));
this.defaultFormat = new Lucene84PostingsFormat();
}

@Override
public StoredFieldsFormat storedFieldsFormat() {
return storedFieldsFormat;
}

@Override
public TermVectorsFormat termVectorsFormat() {
return vectorsFormat;
}

@Override
public PostingsFormat postingsFormat() {
return postingsFormat;
}

@Override
public final FieldInfosFormat fieldInfosFormat() {
return fieldInfosFormat;
}

@Override
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}

@Override
public final LiveDocsFormat liveDocsFormat() {
return liveDocsFormat;
}

@Override
public CompoundFormat compoundFormat() {
return compoundFormat;
}

@Override
public PointsFormat pointsFormat() {
return pointsFormat;
}

@Override
public final KnnVectorsFormat knnVectorsFormat() {
return KnnVectorsFormat.EMPTY;
}

/**
* Returns the postings format that should be used for writing new segments of <code>field</code>.
*
* <p>The default implementation always returns "Lucene84".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation.
*/
public PostingsFormat getPostingsFormatForField(String field) {
return defaultFormat;
}

/**
* Returns the docvalues format that should be used for writing new segments of <code>field</code>
* .
*
* <p>The default implementation always returns "Lucene80".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation.
*/
public DocValuesFormat getDocValuesFormatForField(String field) {
return defaultDVFormat;
}

@Override
public final DocValuesFormat docValuesFormat() {
return docValuesFormat;
}

private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80");

private final NormsFormat normsFormat = new Lucene80NormsFormat();

@Override
public NormsFormat normsFormat() {
return normsFormat;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.lucene.bwc.codecs.lucene87;

import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;
import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;
import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat;
import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat;
import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat;
import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;

public class BWCLucene87Codec extends BWCCodec {

private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat());
private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene86SegmentInfoFormat());
private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
private final PointsFormat pointsFormat = new Lucene86PointsFormat();
private final PostingsFormat defaultFormat;

private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return BWCLucene87Codec.this.getPostingsFormatForField(field);
}
};

private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
return BWCLucene87Codec.this.getDocValuesFormatForField(field);
}
};

private final StoredFieldsFormat storedFieldsFormat;

/** Instantiates a new codec. */
public BWCLucene87Codec() {
super("BWCLucene87Codec");
this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION);
this.defaultFormat = new Lucene84PostingsFormat();
this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION);
}

@Override
public StoredFieldsFormat storedFieldsFormat() {
return storedFieldsFormat;
}

@Override
public TermVectorsFormat termVectorsFormat() {
return vectorsFormat;
}

@Override
public PostingsFormat postingsFormat() {
return postingsFormat;
}

@Override
public final FieldInfosFormat fieldInfosFormat() {
return fieldInfosFormat;
}

@Override
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}

@Override
public final LiveDocsFormat liveDocsFormat() {
return liveDocsFormat;
}

@Override
public CompoundFormat compoundFormat() {
return compoundFormat;
}

@Override
public PointsFormat pointsFormat() {
return pointsFormat;
}

@Override
public final KnnVectorsFormat knnVectorsFormat() {
return KnnVectorsFormat.EMPTY;
}

/**
* Returns the postings format that should be used for writing new segments of <code>field</code>.
*
* <p>The default implementation always returns "Lucene84".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation.
*/
public PostingsFormat getPostingsFormatForField(String field) {
return defaultFormat;
}

/**
* Returns the docvalues format that should be used for writing new segments of <code>field</code>
* .
*
* <p>The default implementation always returns "Lucene80".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation.
*/
public DocValuesFormat getDocValuesFormatForField(String field) {
return defaultDVFormat;
}

@Override
public final DocValuesFormat docValuesFormat() {
return docValuesFormat;
}

private final DocValuesFormat defaultDVFormat;

private final NormsFormat normsFormat = new Lucene80NormsFormat();

@Override
public NormsFormat normsFormat() {
return normsFormat;
}

}
Original file line number Diff line number Diff line change
@@ -5,6 +5,8 @@
# 2.0.
#

org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec
org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec
org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec
org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70Codec
org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec
Loading