diff --git a/.github/workflows/github-merit-badger.yml b/.github/workflows/github-merit-badger.yml deleted file mode 100644 index ee00e62da2f08..0000000000000 --- a/.github/workflows/github-merit-badger.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: github-merit-badger -on: - pull_request_target: - types: - - opened - -jobs: - call-action: - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - uses: aws-github-ops/github-merit-badger@v0.0.98 - id: merit-badger - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - badges: '[first-time-contributor,repeat-contributor,valued-contributor,seasoned-contributor,all-star-contributor,distinguished-contributor]' - thresholds: '[0,3,6,13,25,50]' - badge-type: 'achievement' - ignore-usernames: '[opensearch-ci-bot, dependabot, opensearch-trigger-bot]' diff --git a/CHANGELOG.md b/CHANGELOG.md index 2501b0ae6ffde..f0ded8bd0fe36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,9 +38,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.maxmind.geoip2:geoip2` from 4.0.0 to 4.0.1 - Bump `com.networknt:json-schema-validator` from 1.0.76 to 1.0.78 - Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.0.1 -- Bump `org.apache.zookeeper:zookeeper` from 3.8.0 to 3.8.1 -- Bump `net.minidev:json-smart` from 2.4.8 to 2.4.10 -- Bump `org.apache.maven:maven-model` from 3.8.6 to 3.9.1 ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) @@ -66,6 +63,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove LegacyESVersion.V_7_6_ and V_7_7_ Constants ([#4837](https://github.com/opensearch-project/OpenSearch/pull/4837)) - Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018)) - Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021)) +- Remove custom Map, List and Set collection classes ([#6871](https://github.com/opensearch-project/OpenSearch/pull/6871)) ### Fixed - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827)) @@ -89,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Segment Replication] Add new cluster setting to set replication strategy by default for all indices in cluster. ([#6791](https://github.com/opensearch-project/OpenSearch/pull/6791)) - Enable sort optimization for all NumericTypes ([#6464](https://github.com/opensearch-project/OpenSearch/pull/6464) - Remove 'cluster_manager' role attachment when using 'node.master' deprecated setting ([#6331](https://github.com/opensearch-project/OpenSearch/pull/6331)) +- Add new cluster settings to ignore weighted round-robin routing and fallback to default behaviour. ([#6834](https://github.com/opensearch-project/OpenSearch/pull/6834)) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.18.0 to 2.20.0 ([#6490](https://github.com/opensearch-project/OpenSearch/pull/6490)) @@ -100,6 +99,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.protobuf:protobuf-java` from 3.22.0 to 3.22.2 - Bump Netty to 4.1.90.Final ([#6677](https://github.com/opensearch-project/OpenSearch/pull/6677) - Bump `com.diffplug.spotless` from 6.15.0 to 6.17.0 +- Bump `org.apache.zookeeper:zookeeper` from 3.8.0 to 3.8.1 +- Bump `net.minidev:json-smart` from 2.4.7 to 2.4.10 +- Bump `org.apache.maven:maven-model` from 3.6.2 to 3.9.1 +- Bump `org.codehaus.jettison:jettison` from 1.5.3 to 1.5.4 ([#6878](https://github.com/opensearch-project/OpenSearch/pull/6878)) ### Changed - Require MediaType in Strings.toString API ([#6009](https://github.com/opensearch-project/OpenSearch/pull/6009)) diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java new file mode 100644 index 0000000000000..03d541dbb7de5 --- /dev/null +++ b/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java @@ -0,0 +1,150 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.benchmark.store.remote.filecache; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; + +import org.apache.lucene.store.IndexInput; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.opensearch.index.store.remote.filecache.CachedIndexInput; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.index.store.remote.filecache.FileCacheFactory; + +/** + * Simple benchmark test of {@link FileCache}. It uses a uniform random distribution + * of keys, which is very simple but unlikely to be representative of any real life + * workload. + */ +@Warmup(iterations = 1) +@Measurement(iterations = 1) +@Fork(1) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Threads(8) +@SuppressWarnings("unused") // invoked by benchmarking framework +public class FileCacheBenchmark { + private static final CachedIndexInput INDEX_INPUT = new FixedSizeStubIndexInput(); + + @Benchmark + public void get(CacheParameters parameters, Blackhole blackhole) { + blackhole.consume(parameters.fileCache.get(randomKeyInCache(parameters))); + } + + @Benchmark + public void replace(CacheParameters parameters, Blackhole blackhole) { + blackhole.consume(parameters.fileCache.put(randomKeyInCache(parameters), INDEX_INPUT)); + } + + @Benchmark + public void put(CacheParameters parameters, Blackhole blackhole) { + blackhole.consume(parameters.fileCache.put(randomKeyNotInCache(parameters), INDEX_INPUT)); + } + + @Benchmark + public void remove(CacheParameters parameters) { + parameters.fileCache.remove(randomKeyInCache(parameters)); + } + + private static Path randomKeyInCache(CacheParameters parameters) { + int i = ThreadLocalRandom.current().nextInt(parameters.maximumNumberOfEntries); + return Paths.get(Integer.toString(i)); + } + + private static Path randomKeyNotInCache(CacheParameters parameters) { + int i = ThreadLocalRandom.current().nextInt(parameters.maximumNumberOfEntries, parameters.maximumNumberOfEntries * 2); + return Paths.get(Integer.toString(i)); + } + + @State(Scope.Benchmark) + public static class CacheParameters { + @Param({ "65536", "1048576" }) + int maximumNumberOfEntries; + + @Param({ "1", "8" }) + int concurrencyLevel; + + FileCache fileCache; + + @Setup + public void setup() { + fileCache = FileCacheFactory.createConcurrentLRUFileCache( + (long) maximumNumberOfEntries * INDEX_INPUT.length(), + concurrencyLevel + ); + for (long i = 0; i < maximumNumberOfEntries; i++) { + final Path key = Paths.get(Long.toString(i)); + fileCache.put(key, INDEX_INPUT); + fileCache.decRef(key); + } + } + } + + /** + * Stubbed out IndexInput that does nothing but report a fixed size + */ + private static class FixedSizeStubIndexInput extends CachedIndexInput { + private FixedSizeStubIndexInput() { + super(FixedSizeStubIndexInput.class.getSimpleName()); + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void close() {} + + @Override + public long getFilePointer() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(long pos) { + throw new UnsupportedOperationException(); + } + + @Override + public long length() { + return 1024 * 1024 * 8; // 8MiB + } + + @Override + public IndexInput slice(String sliceDescription, long offset, long length) { + throw new UnsupportedOperationException(); + } + + @Override + public byte readByte() { + throw new UnsupportedOperationException(); + } + + @Override + public void readBytes(byte[] b, int offset, int len) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 21070972d2a23..3a1013c01dd53 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -114,7 +114,7 @@ dependencies { api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2' api 'org.jdom:jdom2:2.0.6.1' api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}" - api 'de.thetaphi:forbiddenapis:3.4' + api 'de.thetaphi:forbiddenapis:3.5.1' api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.11' api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}" api 'org.apache.maven:maven-model:3.9.1' diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java index b2d12c4fc2f79..d83f1b01ee043 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -51,7 +51,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { public TaskProvider createTask(Project project) { project.getPlugins().apply(CompileOnlyResolvePlugin.class); project.getConfigurations().create("forbiddenApisCliJar"); - project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.4"); + project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.5.1"); Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME); if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-core") == false) { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java index 88af1ef8c94e3..6139291b9be1b 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java @@ -79,9 +79,7 @@ @CacheableTask public class ThirdPartyAuditTask extends DefaultTask { - private static final Pattern MISSING_CLASS_PATTERN = Pattern.compile( - "WARNING: Class '(.*)' cannot be loaded \\(.*\\)\\. Please fix the classpath!" - ); + private static final Pattern MISSING_CLASS_PATTERN = Pattern.compile("DEBUG: Class '(.*)' cannot be loaded \\(.*\\)\\."); private static final Pattern VIOLATION_PATTERN = Pattern.compile("\\s\\sin ([a-zA-Z0-9$.]+) \\(.*\\)"); private static final int SIG_KILL_EXIT_VALUE = 137; @@ -367,7 +365,7 @@ private String runForbiddenAPIsCli() throws IOException { spec.jvmArgs("-Xmx1g"); spec.jvmArgs(LoggedExec.shortLivedArgs()); spec.getMainClass().set("de.thetaphi.forbiddenapis.cli.CliMain"); - spec.args("-f", getSignatureFile().getAbsolutePath(), "-d", getJarExpandDir(), "--allowmissingclasses"); + spec.args("-f", getSignatureFile().getAbsolutePath(), "-d", getJarExpandDir(), "--debug", "--allowmissingclasses"); spec.setErrorOutput(errorOut); if (getLogger().isInfoEnabled() == false) { spec.setStandardOutput(new NullOutputStream()); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index 3bb92d826e5eb..57bb524bcf607 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -119,8 +119,8 @@ public void apply(Project project) { TaskProvider depsTask = project.getTasks().register(taskname + "#deps"); depsTask.configure(t -> t.dependsOn(distribution, examplePlugin)); depsTasks.put(taskname, depsTask); - // TODO - suppressing failure temporarily where duplicate tasks are created for docker. - try { + // Avoid duplicate tasks such as docker registered in lifecycleTasks + if (project.getTasksByName(taskname, false).isEmpty()) { TaskProvider destructiveTask = configureTestTask(project, taskname, distribution, t -> { t.onlyIf(t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable); addSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath); @@ -134,8 +134,6 @@ public void apply(Project project) { } destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); lifecycleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask)); - } catch (Exception ex) { - System.out.println(ex.getMessage()); } if ((distribution.getType() == OpenSearchDistribution.Type.DEB || distribution.getType() == OpenSearchDistribution.Type.RPM) diff --git a/buildSrc/src/testKit/thirdPartyAudit/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/build.gradle index 537bf3c1fad71..553ff5d8e6ed2 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/build.gradle @@ -40,7 +40,6 @@ repositories { } dependencies { - forbiddenApisCliJar 'de.thetaphi:forbiddenapis:3.4' jdkJarHell 'org.opensearch:opensearch-core:current' compileOnly "org.${project.properties.compileOnlyGroup}:${project.properties.compileOnlyVersion}" implementation "org.${project.properties.compileGroup}:${project.properties.compileVersion}" diff --git a/buildSrc/version.properties b/buildSrc/version.properties index f2a91c6740f5b..a7a1f8717abc3 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.5.0 +lucene = 9.6.0-snapshot-a3ae27f bundled_jdk_vendor = adoptium bundled_jdk = 19.0.2+7 @@ -17,18 +17,18 @@ supercsv = 2.4.0 log4j = 2.17.1 slf4j = 1.7.36 asm = 9.4 -jettison = 1.5.3 +jettison = 1.5.4 woodstox = 6.4.0 kotlin = 1.7.10 antlr4 = 4.11.1 guava = 31.1-jre - + # when updating the JNA version, also update the version in buildSrc/build.gradle jna = 5.5.0 netty = 4.1.90.Final joda = 2.12.2 - + # client dependencies httpclient5 = 5.1.4 httpcore5 = 5.1.5 diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java index bd143c4a8aef7..c1e2420d86d20 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java @@ -55,7 +55,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.ingest.Pipeline; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchModule; diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index 58beba16820c6..936fd032f36de 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -31,7 +31,7 @@ package org.opensearch.bootstrap; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.settings.KeyStoreCommandTestCase; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreCommandTestCase.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreCommandTestCase.java index 32618923498ff..b87d08bb08399 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreCommandTestCase.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreCommandTestCase.java @@ -42,7 +42,7 @@ import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.cli.CommandTestCase; import org.opensearch.common.io.PathUtilsForTesting; diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java index 70046c567b00e..7a0e286d2d961 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java @@ -39,7 +39,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.NIOFSDirectory; import org.opensearch.common.Randomness; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java index 5bf0bc7763ddd..5a93165b1d992 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java @@ -58,7 +58,7 @@ import org.opensearch.common.SuppressForbidden; import org.opensearch.common.collect.Tuple; import org.opensearch.common.hash.MessageDigests; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import java.io.BufferedReader; diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/PluginCli.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/PluginCli.java index 9b06235e87e86..3ea8e539cdd5c 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/PluginCli.java +++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/PluginCli.java @@ -32,7 +32,7 @@ package org.opensearch.plugins; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.cli.Command; import org.opensearch.cli.LoggingAwareMultiCommand; import org.opensearch.cli.Terminal; diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/RemovePluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/RemovePluginCommand.java index 809c8172bae1c..4ac99c816717a 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/RemovePluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/RemovePluginCommand.java @@ -38,7 +38,7 @@ import org.opensearch.cli.ExitCodes; import org.opensearch.cli.Terminal; import org.opensearch.cli.UserException; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import java.io.IOException; diff --git a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/upgrade/UpgradeCliTests.java b/distribution/tools/upgrade-cli/src/test/java/org/opensearch/upgrade/UpgradeCliTests.java index 39fa2cef24bb1..ee871e1d89001 100644 --- a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/upgrade/UpgradeCliTests.java +++ b/distribution/tools/upgrade-cli/src/test/java/org/opensearch/upgrade/UpgradeCliTests.java @@ -18,7 +18,7 @@ import org.opensearch.common.io.PathUtilsForTesting; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; diff --git a/libs/cli/build.gradle b/libs/cli/build.gradle index 2ce9cc86cf69a..802bd7a774af7 100644 --- a/libs/cli/build.gradle +++ b/libs/cli/build.gradle @@ -34,7 +34,6 @@ apply plugin: 'opensearch.publish' dependencies { api 'net.sf.jopt-simple:jopt-simple:5.0.4' api project(':libs:opensearch-common') - api project(':libs:opensearch-core') } test.enabled = false diff --git a/libs/cli/src/main/java/org/opensearch/cli/MultiCommand.java b/libs/cli/src/main/java/org/opensearch/cli/MultiCommand.java index 1bdcd6189c757..837a93ea95f83 100644 --- a/libs/cli/src/main/java/org/opensearch/cli/MultiCommand.java +++ b/libs/cli/src/main/java/org/opensearch/cli/MultiCommand.java @@ -36,7 +36,7 @@ import joptsimple.OptionSet; import joptsimple.OptionSpec; import joptsimple.util.KeyValuePair; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.util.ArrayList; diff --git a/libs/common/src/main/java/org/opensearch/common/collect/List.java b/libs/common/src/main/java/org/opensearch/common/collect/List.java deleted file mode 100644 index 07cfc2c019856..0000000000000 --- a/libs/common/src/main/java/org/opensearch/common/collect/List.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.common.collect; - -import java.util.Collection; - -/** - * Java 9 List - * - * - * @opensearch.internal - */ -@Deprecated(forRemoval = true) -public class List { - - /** - * Delegates to the Java9 {@code List.of()} method. - * - * @param the {@code List}'s element type - * @return an empty {@code List} - */ - public static java.util.List of() { - return java.util.List.of(); - } - - /** - * Delegates to the Java9 {@code List.of()} method. - * - * @param the {@code List}'s element type - * @param e1 the single element - * @return a {@code List} containing the specified element - */ - public static java.util.List of(T e1) { - return java.util.List.of(e1); - } - - /** - * Delegates to the Java9 {@code List.of()} method. - * - * @param the {@code List}'s element type - * @param e1 the single element - * @return a {@code List} containing the specified element - */ - public static java.util.List of(T e1, T e2) { - return java.util.List.of(e1, e2); - } - - /** - * Delegates to the Java9 {@code List.of()} method. - * - * @param entries the elements to be contained in the list - * @param the {@code List}'s element type - * @return an unmodifiable list containing the specified elements. - */ - @SafeVarargs - @SuppressWarnings("varargs") - public static java.util.List of(T... entries) { - return java.util.List.of(entries); - } - - /** - * Delegates to the Java9 {@code List.copyOf()} method. - * - * @param the {@code List}'s element type - * @param coll a {@code Collection} from which elements are drawn, must be non-null - * @return a {@code List} containing the elements of the given {@code Collection} - */ - public static java.util.List copyOf(Collection coll) { - return java.util.List.copyOf(coll); - } -} diff --git a/libs/common/src/main/java/org/opensearch/common/collect/Map.java b/libs/common/src/main/java/org/opensearch/common/collect/Map.java deleted file mode 100644 index 3913c0fd942a4..0000000000000 --- a/libs/common/src/main/java/org/opensearch/common/collect/Map.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.common.collect; - -/** - * Java 9 Map - * - * - * @opensearch.internal - */ -@Deprecated(forRemoval = true) -public class Map { - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of() { - return java.util.Map.of(); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of(K k1, V v1) { - return java.util.Map.of(k1, v1); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of(K k1, V v1, K k2, V v2) { - return java.util.Map.of(k1, v1, k2, v2); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of(K k1, V v1, K k2, V v2, K k3, V v3) { - return java.util.Map.of(k1, v1, k2, v2, k3, v3); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4) { - return java.util.Map.of(k1, v1, k2, v2, k3, v3, k4, v4); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4, K k5, V v5) { - return java.util.Map.of(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4, K k5, V v5, K k6, V v6) { - return java.util.Map.of(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4, K k5, V v5, K k6, V v6, K k7, V v7) { - return java.util.Map.of(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of( - K k1, - V v1, - K k2, - V v2, - K k3, - V v3, - K k4, - V v4, - K k5, - V v5, - K k6, - V v6, - K k7, - V v7, - K k8, - V v8 - ) { - return java.util.Map.of(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of( - K k1, - V v1, - K k2, - V v2, - K k3, - V v3, - K k4, - V v4, - K k5, - V v5, - K k6, - V v6, - K k7, - V v7, - K k8, - V v8, - K k9, - V v9 - ) { - return java.util.Map.of(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9); - } - - /** - * Delegates to the Java9 {@code Map.of()} method. - */ - public static java.util.Map of( - K k1, - V v1, - K k2, - V v2, - K k3, - V v3, - K k4, - V v4, - K k5, - V v5, - K k6, - V v6, - K k7, - V v7, - K k8, - V v8, - K k9, - V v9, - K k10, - V v10 - ) { - return java.util.Map.of(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10); - } - - /** - * Delegates to the Java9 {@code Map.ofEntries()} method. - */ - @SafeVarargs - @SuppressWarnings("varargs") - public static java.util.Map ofEntries(java.util.Map.Entry... entries) { - return java.util.Map.ofEntries(entries); - } - - /** - * Delegates to the Java9 {@code Map.entry()} method. - */ - public static java.util.Map.Entry entry(K k, V v) { - return java.util.Map.entry(k, v); - } - - /** - * Delegates to the Java10 {@code Map.copyOf()} method. - */ - public static java.util.Map copyOf(java.util.Map map) { - return java.util.Map.copyOf(map); - } - -} diff --git a/server/src/main/java/org/opensearch/common/collect/MapBuilder.java b/libs/common/src/main/java/org/opensearch/common/collect/MapBuilder.java similarity index 100% rename from server/src/main/java/org/opensearch/common/collect/MapBuilder.java rename to libs/common/src/main/java/org/opensearch/common/collect/MapBuilder.java diff --git a/libs/common/src/main/java/org/opensearch/common/collect/Set.java b/libs/common/src/main/java/org/opensearch/common/collect/Set.java deleted file mode 100644 index 0c3899b2aaacd..0000000000000 --- a/libs/common/src/main/java/org/opensearch/common/collect/Set.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.common.collect; - -import java.util.Collection; - -/** - * Java 9 Set - * - * - * @opensearch.internal - */ -@Deprecated(forRemoval = true) -public class Set { - - /** - * Delegates to the Java9 {@code Set.of()} method. - * - * @param the {@code Set}'s element type - * @return an empty {@code Set} - */ - public static java.util.Set of() { - return java.util.Set.of(); - } - - /** - * Delegates to the Java9 {@code Set.of()} method. - * - * @param the {@code Set}'s element type - * @param e1 the single element - * @return a {@code Set} containing the specified element - */ - public static java.util.Set of(T e1) { - return java.util.Set.of(e1); - } - - /** - * Delegates to the Java9 {@code Set.of()} method. - * - * @param the {@code Set}'s element type - * @param e1 the first element - * @param e2 the second element - * @return a {@code Set} containing the specified element - */ - public static java.util.Set of(T e1, T e2) { - return java.util.Set.of(e1, e2); - } - - /** - * Delegates to the Java9 {@code Set.of()} method. - * - * @param entries the elements to be contained in the set - * @param the {@code Set}'s element type - * @return an unmodifiable set containing the specified elements. - */ - @SafeVarargs - @SuppressWarnings("varargs") - public static java.util.Set of(T... entries) { - return java.util.Set.of(entries); - } - - /** - * Delegates to the Java10 {@code Set.copyOf} method. - * - * @param the {@code Set}'s element type - * @param coll a {@code Collection} from which elements are drawn, must be non-null - * @return a {@code Set} containing the elements of the given {@code Collection} - */ - public static java.util.Set copyOf(Collection coll) { - return java.util.Set.copyOf(coll); - } -} diff --git a/libs/core/src/main/java/org/opensearch/core/internal/io/IOUtils.java b/libs/common/src/main/java/org/opensearch/common/util/io/IOUtils.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/core/internal/io/IOUtils.java rename to libs/common/src/main/java/org/opensearch/common/util/io/IOUtils.java index 25494ca440ef4..6b72a0f92c14d 100644 --- a/libs/core/src/main/java/org/opensearch/core/internal/io/IOUtils.java +++ b/libs/common/src/main/java/org/opensearch/common/util/io/IOUtils.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.core.internal.io; +package org.opensearch.common.util.io; import org.opensearch.common.Nullable; diff --git a/libs/core/src/main/java/org/opensearch/core/internal/io/Streams.java b/libs/common/src/main/java/org/opensearch/common/util/io/Streams.java similarity index 98% rename from libs/core/src/main/java/org/opensearch/core/internal/io/Streams.java rename to libs/common/src/main/java/org/opensearch/common/util/io/Streams.java index a54e23f18d1a5..96672a3be3dc1 100644 --- a/libs/core/src/main/java/org/opensearch/core/internal/io/Streams.java +++ b/libs/common/src/main/java/org/opensearch/common/util/io/Streams.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.core.internal.io; +package org.opensearch.common.util.io; import java.io.IOException; import java.io.InputStream; diff --git a/libs/core/src/main/java/org/opensearch/core/internal/io/package-info.java b/libs/common/src/main/java/org/opensearch/common/util/io/package-info.java similarity index 71% rename from libs/core/src/main/java/org/opensearch/core/internal/io/package-info.java rename to libs/common/src/main/java/org/opensearch/common/util/io/package-info.java index 6bcb414ab584e..5789028538a48 100644 --- a/libs/core/src/main/java/org/opensearch/core/internal/io/package-info.java +++ b/libs/common/src/main/java/org/opensearch/common/util/io/package-info.java @@ -6,5 +6,5 @@ * compatible open source license. */ -/** Common internal I/O classes */ -package org.opensearch.core.internal.io; +/** Common internal I/O utility classes */ +package org.opensearch.common.util.io; diff --git a/libs/core/src/main/java/org/opensearch/core/internal/net/NetUtils.java b/libs/common/src/main/java/org/opensearch/common/util/net/NetUtils.java similarity index 99% rename from libs/core/src/main/java/org/opensearch/core/internal/net/NetUtils.java rename to libs/common/src/main/java/org/opensearch/common/util/net/NetUtils.java index a77efb81e5bd0..708b62c290949 100644 --- a/libs/core/src/main/java/org/opensearch/core/internal/net/NetUtils.java +++ b/libs/common/src/main/java/org/opensearch/common/util/net/NetUtils.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.core.internal.net; +package org.opensearch.common.util.net; import java.io.IOException; import java.lang.reflect.Field; diff --git a/libs/core/src/main/java/org/opensearch/core/internal/net/package-info.java b/libs/common/src/main/java/org/opensearch/common/util/net/package-info.java similarity index 85% rename from libs/core/src/main/java/org/opensearch/core/internal/net/package-info.java rename to libs/common/src/main/java/org/opensearch/common/util/net/package-info.java index 2b530f88aedbf..126b0939eab32 100644 --- a/libs/core/src/main/java/org/opensearch/core/internal/net/package-info.java +++ b/libs/common/src/main/java/org/opensearch/common/util/net/package-info.java @@ -7,4 +7,4 @@ */ /** Common Network Utility Classes */ -package org.opensearch.core.internal.net; +package org.opensearch.common.util.net; diff --git a/libs/core/build.gradle b/libs/core/build.gradle index 5e27c15c7477f..717320b95ac85 100644 --- a/libs/core/build.gradle +++ b/libs/core/build.gradle @@ -79,6 +79,9 @@ dependencies { api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + // lucene + api "org.apache.lucene:lucene-core:${versions.lucene}" + testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" testImplementation "junit:junit:${versions.junit}" testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" @@ -96,4 +99,5 @@ tasks.named('forbiddenApisMain').configure { tasks.named("dependencyLicenses").configure { mapping from: /jackson-.*/, to: 'jackson' + mapping from: /lucene-.*/, to: 'lucene' } diff --git a/libs/core/licenses/lucene-LICENSE.txt b/libs/core/licenses/lucene-LICENSE.txt new file mode 100644 index 0000000000000..28b134f5f8e4d --- /dev/null +++ b/libs/core/licenses/lucene-LICENSE.txt @@ -0,0 +1,475 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from unicode conversion examples available at +http://www.unicode.org/Public/PROGRAMS/CVTUTF. Here is the copyright +from those sources: + +/* + * Copyright 2001-2004 Unicode, Inc. + * + * Disclaimer + * + * This source code is provided as is by Unicode, Inc. No claims are + * made as to fitness for any particular purpose. No warranties of any + * kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been + * purchased on magnetic or optical media from Unicode, Inc., the + * sole remedy for any claim will be exchange of defective media + * within 90 days of receipt. + * + * Limitations on Rights to Redistribute This Code + * + * Unicode, Inc. hereby grants the right to freely use the information + * supplied in this file in the creation of products supporting the + * Unicode Standard, and to make copies of this file in any form + * for internal or external distribution as long as this notice + * remains attached. + */ + + +Some code in core/src/java/org/apache/lucene/util/ArrayUtil.java was +derived from Python 2.4.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/2.4.2/license/ + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from Python 3.1.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/3.1.2/license/ + +Some code in core/src/java/org/apache/lucene/util/automaton was +derived from Brics automaton sources available at +www.brics.dk/automaton/. Here is the copyright from those sources: + +/* + * Copyright (c) 2001-2009 Anders Moeller + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +The levenshtein automata tables in core/src/java/org/apache/lucene/util/automaton +were automatically generated with the moman/finenight FSA package. +Here is the copyright for those sources: + +# Copyright (c) 2010, Jean-Philippe Barrette-LaPierre, +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from ICU (http://www.icu-project.org) +The full license is available here: + http://source.icu-project.org/repos/icu/icu/trunk/license.html + +/* + * Copyright (C) 1999-2010, International Business Machines + * Corporation and others. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * provided that the above copyright notice(s) and this permission notice appear + * in all copies of the Software and that both the above copyright notice(s) and + * this permission notice appear in supporting documentation. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE + * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR + * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER + * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT + * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall not + * be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + */ + +The following license applies to the Snowball stemmers: + +Copyright (c) 2001, Dr Martin Porter +Copyright (c) 2002, Richard Boulton +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holders nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The following license applies to the KStemmer: + +Copyright © 2003, +Center for Intelligent Information Retrieval, +University of Massachusetts, Amherst. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. The names "Center for Intelligent Information Retrieval" and +"University of Massachusetts" must not be used to endorse or promote products +derived from this software without prior written permission. To obtain +permission, contact info@ciir.cs.umass.edu. + +THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +The following license applies to the Morfologik project: + +Copyright (c) 2006 Dawid Weiss +Copyright (c) 2007-2011 Dawid Weiss, Marcin Miłkowski +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Morfologik nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +The dictionary comes from Morfologik project. Morfologik uses data from +Polish ispell/myspell dictionary hosted at http://www.sjp.pl/slownik/en/ and +is licenced on the terms of (inter alia) LGPL and Creative Commons +ShareAlike. The part-of-speech tags were added in Morfologik project and +are not found in the data from sjp.pl. The tagset is similar to IPI PAN +tagset. + +--- + +The following license applies to the Morfeusz project, +used by org.apache.lucene.analysis.morfologik. + +BSD-licensed dictionary of Polish (SGJP) +http://sgjp.pl/morfeusz/ + +Copyright © 2011 Zygmunt Saloni, Włodzimierz Gruszczyński, + Marcin Woliński, Robert Wołosz + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS “AS IS” AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/libs/core/licenses/lucene-NOTICE.txt b/libs/core/licenses/lucene-NOTICE.txt new file mode 100644 index 0000000000000..1a1d51572432a --- /dev/null +++ b/libs/core/licenses/lucene-NOTICE.txt @@ -0,0 +1,192 @@ +Apache Lucene +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Includes software from other Apache Software Foundation projects, +including, but not limited to: + - Apache Ant + - Apache Jakarta Regexp + - Apache Commons + - Apache Xerces + +ICU4J, (under analysis/icu) is licensed under an MIT styles license +and Copyright (c) 1995-2008 International Business Machines Corporation and others + +Some data files (under analysis/icu/src/data) are derived from Unicode data such +as the Unicode Character Database. See http://unicode.org/copyright.html for more +details. + +Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is +BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ + +The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were +automatically generated with the moman/finenight FSA library, created by +Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, +see http://sites.google.com/site/rrettesite/moman and +http://bitbucket.org/jpbarrette/moman/overview/ + +The class org.apache.lucene.util.WeakIdentityMap was derived from +the Apache CXF project and is Apache License 2.0. + +The Google Code Prettify is Apache License 2.0. +See http://code.google.com/p/google-code-prettify/ + +JUnit (junit-4.10) is licensed under the Common Public License v. 1.0 +See http://junit.sourceforge.net/cpl-v10.html + +This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin +g Package (jaspell): http://jaspell.sourceforge.net/ +License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) + +The snowball stemmers in + analysis/common/src/java/net/sf/snowball +were developed by Martin Porter and Richard Boulton. +The snowball stopword lists in + analysis/common/src/resources/org/apache/lucene/analysis/snowball +were developed by Martin Porter and Richard Boulton. +The full snowball package is available from + http://snowball.tartarus.org/ + +The KStem stemmer in + analysis/common/src/org/apache/lucene/analysis/en +was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) +under the BSD-license. + +The Arabic,Persian,Romanian,Bulgarian, Hindi and Bengali analyzers (common) come with a default +stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: +analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bn/stopwords.txt +See http://members.unine.ch/jacques.savoy/clef/index.html. + +The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers +(common) are based on BSD-licensed reference implementations created by Jacques Savoy and +Ljiljana Dolamic. These files reside in: +analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java + +The Stempel analyzer (stempel) includes BSD-licensed software developed +by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, +and Edmond Nolan. + +The Polish analyzer (stempel) comes with a default +stopword list that is BSD-licensed created by the Carrot2 project. The file resides +in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. +See http://project.carrot2.org/license.html. + +The SmartChineseAnalyzer source code (smartcn) was +provided by Xiaoping Gao and copyright 2009 by www.imdict.net. + +WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/) +is derived from Unicode data such as the Unicode Character Database. +See http://unicode.org/copyright.html for more details. + +The Morfologik analyzer (morfologik) includes BSD-licensed software +developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/). + +Morfologik uses data from Polish ispell/myspell dictionary +(http://www.sjp.pl/slownik/en/) licenced on the terms of (inter alia) +LGPL and Creative Commons ShareAlike. + +Morfologic includes data from BSD-licensed dictionary of Polish (SGJP) +(http://sgjp.pl/morfeusz/) + +Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original +source code for this can be found at http://www.eclipse.org/jetty/downloads.php + +=========================================================================== +Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration +=========================================================================== + +This software includes a binary and/or source version of data from + + mecab-ipadic-2.7.0-20070801 + +which can be obtained from + + http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz + +or + + http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz + +=========================================================================== +mecab-ipadic-2.7.0-20070801 Notice +=========================================================================== + +Nara Institute of Science and Technology (NAIST), +the copyright holders, disclaims all warranties with regard to this +software, including all implied warranties of merchantability and +fitness, in no event shall NAIST be liable for +any special, indirect or consequential damages or any damages +whatsoever resulting from loss of use, data or profits, whether in an +action of contract, negligence or other tortuous action, arising out +of or in connection with the use or performance of this software. + +A large portion of the dictionary entries +originate from ICOT Free Software. The following conditions for ICOT +Free Software applies to the current dictionary as well. + +Each User may also freely distribute the Program, whether in its +original form or modified, to any third party or parties, PROVIDED +that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear +on, or be attached to, the Program, which is distributed substantially +in the same form as set out herein and that such intended +distribution, if actually made, will neither violate or otherwise +contravene any of the laws and regulations of the countries having +jurisdiction over the User or the intended distribution itself. + +NO WARRANTY + +The program was produced on an experimental basis in the course of the +research and development conducted during the project and is provided +to users as so produced on an experimental basis. Accordingly, the +program is provided without any warranty whatsoever, whether express, +implied, statutory or otherwise. The term "warranty" used herein +includes, but is not limited to, any warranty of the quality, +performance, merchantability and fitness for a particular purpose of +the program and the nonexistence of any infringement or violation of +any right of any third party. + +Each user of the program will agree and understand, and be deemed to +have agreed and understood, that there is no warranty whatsoever for +the program and, accordingly, the entire risk arising from or +otherwise connected with the program is assumed by the user. + +Therefore, neither ICOT, the copyright holder, or any other +organization that participated in or was otherwise related to the +development of the program and their respective officials, directors, +officers and other employees shall be held liable for any and all +damages, including, without limitation, general, special, incidental +and consequential damages, arising out of or otherwise in connection +with the use or inability to use the program or any product, material +or result produced or otherwise obtained by using the program, +regardless of whether they have been advised of, or otherwise had +knowledge of, the possibility of such damages at any time during the +project or thereafter. Each user will be deemed to have agreed to the +foregoing by his or her commencement of use of the program. The term +"use" as used herein includes, but is not limited to, the use, +modification, copying and distribution of the program and the +production of secondary products from the program. + +In the case where the program, whether in its original form or +modified, was distributed or delivered to or received by a user from +any person, organization or entity other than ICOT, unless it makes or +grants independently of ICOT any specific warranty to the user in +writing, such person, organization or entity, will also be exempted +from and not be held liable to the user for any such damages as noted +above as far as the program is concerned. diff --git a/server/licenses/lucene-core-9.5.0.jar.sha1 b/libs/core/licenses/lucene-core-9.5.0.jar.sha1 similarity index 100% rename from server/licenses/lucene-core-9.5.0.jar.sha1 rename to libs/core/licenses/lucene-core-9.5.0.jar.sha1 diff --git a/server/src/main/java/org/opensearch/Assertions.java b/libs/core/src/main/java/org/opensearch/core/Assertions.java similarity index 92% rename from server/src/main/java/org/opensearch/Assertions.java rename to libs/core/src/main/java/org/opensearch/core/Assertions.java index dd2fcf9481a3c..da93f2f435441 100644 --- a/server/src/main/java/org/opensearch/Assertions.java +++ b/libs/core/src/main/java/org/opensearch/core/Assertions.java @@ -30,12 +30,12 @@ * GitHub history for details. */ -package org.opensearch; +package org.opensearch.core; /** * Provides a static final field that can be used to check if assertions are enabled. Since this field might be used elsewhere to check if * assertions are enabled, if you are running with assertions enabled for specific packages or classes, you should enable assertions on this - * class too (e.g., {@code -ea org.opensearch.Assertions -ea org.opensearch.cluster.service.ClusterManagerService}). + * class too (e.g., {@code -ea org.opensearch.core.Assertions -ea org.opensearch.cluster.service.ClusterManagerService}). * * @opensearch.internal */ diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchRejectedExecutionException.java b/libs/core/src/main/java/org/opensearch/core/concurrency/OpenSearchRejectedExecutionException.java similarity index 97% rename from server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchRejectedExecutionException.java rename to libs/core/src/main/java/org/opensearch/core/concurrency/OpenSearchRejectedExecutionException.java index 74b5e477c36ee..2a1234eb66c4e 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchRejectedExecutionException.java +++ b/libs/core/src/main/java/org/opensearch/core/concurrency/OpenSearchRejectedExecutionException.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.util.concurrent; +package org.opensearch.core.concurrency; import java.util.concurrent.RejectedExecutionException; diff --git a/libs/core/src/main/java/org/opensearch/core/internal/package-info.java b/libs/core/src/main/java/org/opensearch/core/concurrency/package-info.java similarity index 73% rename from libs/core/src/main/java/org/opensearch/core/internal/package-info.java rename to libs/core/src/main/java/org/opensearch/core/concurrency/package-info.java index c39e17c401a1c..331066995ccd5 100644 --- a/libs/core/src/main/java/org/opensearch/core/internal/package-info.java +++ b/libs/core/src/main/java/org/opensearch/core/concurrency/package-info.java @@ -5,6 +5,5 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ - -/** Common internal core classes */ -package org.opensearch.core.internal; +/** core concurrency package */ +package org.opensearch.core.concurrency; diff --git a/libs/core/src/main/java/org/opensearch/core/util/BytesRefUtils.java b/libs/core/src/main/java/org/opensearch/core/util/BytesRefUtils.java new file mode 100644 index 0000000000000..bfea55479c235 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/util/BytesRefUtils.java @@ -0,0 +1,105 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.core.util; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefArray; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.InPlaceMergeSorter; + +import java.util.Comparator; + +/** + * Utilities for sorting Lucene {@link BytesRefArray} + * + * @opensearch.internal + */ +public class BytesRefUtils { + public static void sort(final BytesRefArray bytes, final int[] indices) { + sort(new BytesRefBuilder(), new BytesRefBuilder(), bytes, indices); + } + + private static void sort( + final BytesRefBuilder scratch, + final BytesRefBuilder scratch1, + final BytesRefArray bytes, + final int[] indices + ) { + + final int numValues = bytes.size(); + assert indices.length >= numValues; + if (numValues > 1) { + new InPlaceMergeSorter() { + final Comparator comparator = Comparator.naturalOrder(); + + @Override + protected int compare(int i, int j) { + return comparator.compare(bytes.get(scratch, indices[i]), bytes.get(scratch1, indices[j])); + } + + @Override + protected void swap(int i, int j) { + int value_i = indices[i]; + indices[i] = indices[j]; + indices[j] = value_i; + } + }.sort(0, numValues); + } + + } + + public static int sortAndDedup(final BytesRefArray bytes, final int[] indices) { + final BytesRefBuilder scratch = new BytesRefBuilder(); + final BytesRefBuilder scratch1 = new BytesRefBuilder(); + final int numValues = bytes.size(); + assert indices.length >= numValues; + if (numValues <= 1) { + return numValues; + } + sort(scratch, scratch1, bytes, indices); + int uniqueCount = 1; + BytesRefBuilder previous = scratch; + BytesRefBuilder current = scratch1; + bytes.get(previous, indices[0]); + for (int i = 1; i < numValues; ++i) { + bytes.get(current, indices[i]); + if (!previous.get().equals(current.get())) { + indices[uniqueCount++] = indices[i]; + } + BytesRefBuilder tmp = previous; + previous = current; + current = tmp; + } + return uniqueCount; + } +} diff --git a/libs/core/src/main/java/org/opensearch/core/util/package-info.java b/libs/core/src/main/java/org/opensearch/core/util/package-info.java new file mode 100644 index 0000000000000..deab57b120644 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/util/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Core utility classes that need the Lucene API */ +package org.opensearch.core.util; diff --git a/libs/core/src/test/java/org/opensearch/core/internal/io/IOUtilsTests.java b/libs/core/src/test/java/org/opensearch/core/internal/io/IOUtilsTests.java index e1f3cb7520a7e..cd7a40b655579 100644 --- a/libs/core/src/test/java/org/opensearch/core/internal/io/IOUtilsTests.java +++ b/libs/core/src/test/java/org/opensearch/core/internal/io/IOUtilsTests.java @@ -35,6 +35,7 @@ import org.apache.lucene.util.Constants; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.io.PathUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import java.io.Closeable; diff --git a/libs/core/src/test/java/org/opensearch/core/internal/io/StreamsTests.java b/libs/core/src/test/java/org/opensearch/core/internal/io/StreamsTests.java index b489a9a018ae5..2ba18221cbe4f 100644 --- a/libs/core/src/test/java/org/opensearch/core/internal/io/StreamsTests.java +++ b/libs/core/src/test/java/org/opensearch/core/internal/io/StreamsTests.java @@ -32,6 +32,7 @@ package org.opensearch.core.internal.io; +import org.opensearch.common.util.io.Streams; import org.opensearch.test.OpenSearchTestCase; import java.io.ByteArrayInputStream; diff --git a/libs/core/src/test/java/org/opensearch/core/internal/net/NetUtilsTests.java b/libs/core/src/test/java/org/opensearch/core/internal/net/NetUtilsTests.java index 448c39731def5..c97b74ee0d624 100644 --- a/libs/core/src/test/java/org/opensearch/core/internal/net/NetUtilsTests.java +++ b/libs/core/src/test/java/org/opensearch/core/internal/net/NetUtilsTests.java @@ -33,7 +33,8 @@ package org.opensearch.core.internal.net; import org.apache.lucene.util.Constants; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.net.NetUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import java.lang.Runtime.Version; diff --git a/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java b/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java new file mode 100644 index 0000000000000..98ecbe96ca5fd --- /dev/null +++ b/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.core.util; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefArray; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.Counter; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * Tests for the {@link BytesRefUtils} class + */ +public class BytesRefUtilsTests extends OpenSearchTestCase { + public void testSortAndDedupByteRefArray() { + SortedSet set = new TreeSet<>(); + final int numValues = scaledRandomIntBetween(0, 10000); + List tmpList = new ArrayList<>(); + BytesRefArray array = new BytesRefArray(Counter.newCounter()); + for (int i = 0; i < numValues; i++) { + String s = randomRealisticUnicodeOfCodepointLengthBetween(1, 100); + set.add(new BytesRef(s)); + tmpList.add(new BytesRef(s)); + array.append(new BytesRef(s)); + } + if (randomBoolean()) { + Collections.shuffle(tmpList, random()); + for (BytesRef ref : tmpList) { + array.append(ref); + } + } + int[] indices = new int[array.size()]; + for (int i = 0; i < indices.length; i++) { + indices[i] = i; + } + int numUnique = BytesRefUtils.sortAndDedup(array, indices); + assertThat(numUnique, equalTo(set.size())); + Iterator iterator = set.iterator(); + + BytesRefBuilder spare = new BytesRefBuilder(); + for (int i = 0; i < numUnique; i++) { + assertThat(iterator.hasNext(), is(true)); + assertThat(array.get(spare, indices[i]), equalTo(iterator.next())); + } + } + + public void testSortByteRefArray() { + List values = new ArrayList<>(); + final int numValues = scaledRandomIntBetween(0, 10000); + BytesRefArray array = new BytesRefArray(Counter.newCounter()); + for (int i = 0; i < numValues; i++) { + String s = randomRealisticUnicodeOfCodepointLengthBetween(1, 100); + values.add(new BytesRef(s)); + array.append(new BytesRef(s)); + } + if (randomBoolean()) { + Collections.shuffle(values, random()); + } + int[] indices = new int[array.size()]; + for (int i = 0; i < indices.length; i++) { + indices[i] = i; + } + BytesRefUtils.sort(array, indices); + Collections.sort(values); + Iterator iterator = values.iterator(); + + BytesRefBuilder spare = new BytesRefBuilder(); + for (int i = 0; i < values.size(); i++) { + assertThat(iterator.hasNext(), is(true)); + assertThat(array.get(spare, indices[i]), equalTo(iterator.next())); + } + } +} diff --git a/libs/nio/build.gradle b/libs/nio/build.gradle index a497ace80f664..1bd0c91135330 100644 --- a/libs/nio/build.gradle +++ b/libs/nio/build.gradle @@ -33,7 +33,6 @@ import org.opensearch.gradle.info.BuildParams apply plugin: 'opensearch.publish' dependencies { - api project(':libs:opensearch-core') api project(':libs:opensearch-common') testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" diff --git a/libs/nio/src/main/java/org/opensearch/nio/SocketChannelContext.java b/libs/nio/src/main/java/org/opensearch/nio/SocketChannelContext.java index 9cf57910a2b96..12a1e80055823 100644 --- a/libs/nio/src/main/java/org/opensearch/nio/SocketChannelContext.java +++ b/libs/nio/src/main/java/org/opensearch/nio/SocketChannelContext.java @@ -33,7 +33,7 @@ package org.opensearch.nio; import org.opensearch.common.concurrent.CompletableContext; -import org.opensearch.core.internal.net.NetUtils; +import org.opensearch.common.util.net.NetUtils; import org.opensearch.nio.utils.ByteBufferUtils; import org.opensearch.nio.utils.ExceptionsHelper; diff --git a/libs/nio/src/test/java/org/opensearch/nio/ChannelFactoryTests.java b/libs/nio/src/test/java/org/opensearch/nio/ChannelFactoryTests.java index ba212de7941e5..0e4ec4a1b3bc5 100644 --- a/libs/nio/src/test/java/org/opensearch/nio/ChannelFactoryTests.java +++ b/libs/nio/src/test/java/org/opensearch/nio/ChannelFactoryTests.java @@ -32,7 +32,7 @@ package org.opensearch.nio; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle index 2ace2e03bc09d..3226ec12ff6f7 100644 --- a/libs/ssl-config/build.gradle +++ b/libs/ssl-config/build.gradle @@ -34,7 +34,6 @@ apply plugin: "opensearch.publish" dependencies { api project(':libs:opensearch-common') - api project(':libs:opensearch-core') testImplementation(project(":test:framework")) { exclude group: 'org.opensearch', module: 'opensearch-ssl-config' diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java index d538c20b5767f..9164b5216f9a5 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java @@ -50,7 +50,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.filtering.FilterPathBasedFilter; -import org.opensearch.core.internal.io.Streams; +import org.opensearch.common.util.io.Streams; import java.io.BufferedInputStream; import java.io.IOException; diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentParser.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentParser.java index 8ff212893da87..d11b5630b6656 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentParser.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentParser.java @@ -40,7 +40,7 @@ import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.AbstractXContentParser; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.nio.CharBuffer; diff --git a/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/DatabaseReaderLazyLoader.java index 44b03744b6685..05a969daf3836 100644 --- a/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/DatabaseReaderLazyLoader.java +++ b/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/DatabaseReaderLazyLoader.java @@ -37,7 +37,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.CheckedSupplier; import org.opensearch.common.SetOnce; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.Closeable; import java.io.IOException; diff --git a/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/IngestGeoIpModulePlugin.java b/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/IngestGeoIpModulePlugin.java index 7869e4a7de7fc..491f55147ef24 100644 --- a/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/IngestGeoIpModulePlugin.java +++ b/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/IngestGeoIpModulePlugin.java @@ -43,7 +43,7 @@ import org.opensearch.common.cache.CacheBuilder; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Setting; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.ingest.Processor; import org.opensearch.plugins.IngestPlugin; import org.opensearch.plugins.Plugin; diff --git a/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1 deleted file mode 100644 index c1340a55815e5..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -62f3d3630ecc14069d5c24b9693df5a2787f8202 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.6.0-snapshot-a3ae27f.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..bf30f6bba7714 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +d1d4772f25f73d7dd6fc37677e124d4ea7f96126 \ No newline at end of file diff --git a/modules/lang-painless/src/doc/java/org/opensearch/painless/ContextDocGenerator.java b/modules/lang-painless/src/doc/java/org/opensearch/painless/ContextDocGenerator.java index 64e9a57917e3f..3fca92a216b26 100644 --- a/modules/lang-painless/src/doc/java/org/opensearch/painless/ContextDocGenerator.java +++ b/modules/lang-painless/src/doc/java/org/opensearch/painless/ContextDocGenerator.java @@ -36,7 +36,7 @@ import org.opensearch.common.io.PathUtils; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.painless.action.PainlessContextClassBindingInfo; import org.opensearch.painless.action.PainlessContextClassInfo; import org.opensearch.painless.action.PainlessContextConstructorInfo; diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java index 0baf3db32a4d8..be12c49321b87 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -45,7 +45,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.fielddata.LeafNumericFieldData; import org.opensearch.index.fielddata.SortedNumericDoubleValues; diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java index 77e34560d4cad..3b70455960966 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java @@ -138,7 +138,10 @@ public void testPrefixQuery() { // this term should be too long to be rewriteable to a term query on the prefix field final String longTerm = "toolongforourprefixfieldthistermis"; - assertThat(fieldType.prefixQuery(longTerm, CONSTANT_SCORE_REWRITE, MOCK_QSC), equalTo(new PrefixQuery(new Term(NAME, longTerm)))); + assertThat( + fieldType.prefixQuery(longTerm, CONSTANT_SCORE_REWRITE, MOCK_QSC), + equalTo(new PrefixQuery(new Term(NAME, longTerm), CONSTANT_SCORE_REWRITE)) + ); OpenSearchException ee = expectThrows( OpenSearchException.class, diff --git a/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java index 85110192899ae..f0a89d6ec2939 100644 --- a/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java @@ -36,7 +36,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; @@ -50,9 +49,7 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.Uid; import org.opensearch.index.query.IdsQueryBuilder; import org.opensearch.index.query.InnerHitBuilder; import org.opensearch.index.query.InnerHitContextBuilder; @@ -307,16 +304,9 @@ static void assertLateParsingQuery(Query query, String type, String id) throws I assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class)); TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).getQuery(); Query rewrittenTermsQuery = termsQuery.rewrite(null); - assertThat(rewrittenTermsQuery, instanceOf(ConstantScoreQuery.class)); - ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) rewrittenTermsQuery; - assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class)); - BooleanQuery booleanTermsQuery = (BooleanQuery) constantScoreQuery.getQuery(); - assertThat(booleanTermsQuery.clauses().toString(), booleanTermsQuery.clauses().size(), equalTo(1)); - assertThat(booleanTermsQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD)); - assertThat(booleanTermsQuery.clauses().get(0).getQuery(), instanceOf(TermQuery.class)); - TermQuery termQuery = (TermQuery) booleanTermsQuery.clauses().get(0).getQuery(); - assertThat(termQuery.getTerm().field(), equalTo(IdFieldMapper.NAME)); - assertThat(termQuery.getTerm().bytes(), equalTo(Uid.encodeId(id))); + // The query is of type MultiTermQueryConstantScoreBlendedWrapper and is sealed inside Apache Lucene, + // no access to inner queries without using the reflection, falling back to stringified query comparison + assertThat(rewrittenTermsQuery.toString(), equalTo("_id:([ff 69 64])")); // check the type filter assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER)); assertEquals(new TermQuery(new Term("join_field", type)), booleanQuery.clauses().get(1).getQuery()); diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java index ac3c6ee82a40d..b1e4d53c90d92 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java @@ -39,7 +39,7 @@ import org.opensearch.common.ParsingException; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/AsyncBulkByScrollActionTests.java index 145aaaf24fbee..61336156a83d0 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -75,7 +75,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.Index; import org.opensearch.index.engine.VersionConflictEngineException; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/ClientScrollableHitSourceTests.java index 3a79a0b35e391..25917893645cf 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/ClientScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/ClientScrollableHitSourceTests.java @@ -48,7 +48,7 @@ import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.internal.InternalSearchResponse; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/RetryTests.java index 0a01843405d9f..4dbd18a15b521 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/RetryTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.http.HttpInfo; import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.Plugin; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteResponseParsersTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteResponseParsersTests.java index 68a37f758288d..2a082e496b9e2 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteResponseParsersTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteResponseParsersTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.reindex.remote; import org.opensearch.action.search.ShardSearchFailure; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java index c0e2bd14f55bc..b1dd92b5f6dc2 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java @@ -48,7 +48,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.reindex.RejectAwareActionListener; import org.opensearch.index.reindex.ScrollableHitSource; import org.opensearch.index.reindex.ScrollableHitSource.Response; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java index 0e61362686959..2074279a99f79 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java @@ -83,8 +83,8 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; -import org.opensearch.core.internal.net.NetUtils; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.common.util.net.NetUtils; import org.opensearch.http.AbstractHttpServerTransport; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpHandlingSettings; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java index 45bd5464485d5..444cf420e4110 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java @@ -62,7 +62,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.core.internal.net.NetUtils; +import org.opensearch.common.util.net.NetUtils; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Netty4NioSocketChannel; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java index 848819ee5b840..2546e2fba8c58 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java @@ -41,8 +41,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.core.internal.io.IOUtils; -import org.opensearch.core.internal.net.NetUtils; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.common.util.net.NetUtils; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.transport.MockTransportService; import org.opensearch.test.transport.StubbableTransport; diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1 deleted file mode 100644 index 1d3eb41aab0a1..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11c816250e4ff106151fd8cb69e61ead4fb4a8dd \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0-snapshot-a3ae27f.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..1773ef80461a8 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +3a90a798de88ac1d4a8b0c081e087bc0fd05e8dd \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1 deleted file mode 100644 index 31f1eb3991d5a..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4555fc64ce9d63c6c1f4983e816526f896606e9f \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0-snapshot-a3ae27f.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..c9e43eba25f08 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +dc5b2ec79dd5a88571bf9c371b6faed9e7a7462d \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1 deleted file mode 100644 index 47bfe0e4b667d..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fd55607da8adf8234169d9f4120119c05a8175c \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0-snapshot-a3ae27f.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..cef2d5a91f913 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +9688d5a5f5f942daeba6c43d50cc2bbe1c94c5c7 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1 deleted file mode 100644 index 99ce9a0a4cb9a..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c97163035f1d46249505d15fbdc427b6ae03549a \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0-snapshot-a3ae27f.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..a0b7532fe72df --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +96726c96ab11ba53e7a283b1cb308920cef16ad0 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1 deleted file mode 100644 index e63db9c407a77..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a08ae3e2212837c98d5e1174cf1a293a5c8c0373 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0-snapshot-a3ae27f.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..76ed5cfdda0f4 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +ea7ae79fd9864ae04297c237fea7cd9224e42b5f \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1 deleted file mode 100644 index d2f956a296f96..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -483906d2ebad0ff910d52382c2f027b511788b58 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0-snapshot-a3ae27f.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..72a8c9572651d --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +b5175ad05cf3f6c55750f63569b8b2ca8dd30fca \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1 deleted file mode 100644 index d28a2fa06fe28..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cb00cd193e2f927e6357cf33e0880034571ea528 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0-snapshot-a3ae27f.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..3f53b4d56b572 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +a79e5a8c0d2cf6dc8c76d05914c39df2cba69b01 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/jettison-1.5.3.jar.sha1 b/plugins/discovery-azure-classic/licenses/jettison-1.5.3.jar.sha1 deleted file mode 100644 index afd13439e739c..0000000000000 --- a/plugins/discovery-azure-classic/licenses/jettison-1.5.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -964d35bbdecbbc33cf2f2044e8a1648d9f6f1474 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/jettison-1.5.4.jar.sha1 b/plugins/discovery-azure-classic/licenses/jettison-1.5.4.jar.sha1 new file mode 100644 index 0000000000000..a87b7691bfce8 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/jettison-1.5.4.jar.sha1 @@ -0,0 +1 @@ +174ca56c411b05aec323d8f53e66709c0d28b337 \ No newline at end of file diff --git a/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2NameResolver.java b/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2NameResolver.java index 9fa479e90c956..093ea1eef3ab3 100644 --- a/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2NameResolver.java +++ b/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2NameResolver.java @@ -38,7 +38,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.network.NetworkService.CustomNameResolver; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.BufferedReader; import java.io.IOException; diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/AbstractEC2MockAPITestCase.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/AbstractEC2MockAPITestCase.java index 51b48a8a45f21..3256a01838169 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/AbstractEC2MockAPITestCase.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/AbstractEC2MockAPITestCase.java @@ -39,7 +39,7 @@ import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; diff --git a/plugins/discovery-gce/src/main/java/org/opensearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/opensearch/plugin/discovery/gce/GceDiscoveryPlugin.java index 282361c893689..d4df6d94c061b 100644 --- a/plugins/discovery-gce/src/main/java/org/opensearch/plugin/discovery/gce/GceDiscoveryPlugin.java +++ b/plugins/discovery-gce/src/main/java/org/opensearch/plugin/discovery/gce/GceDiscoveryPlugin.java @@ -46,7 +46,7 @@ import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.discovery.SeedHostsProvider; import org.opensearch.discovery.gce.GceSeedHostsProvider; import org.opensearch.plugins.DiscoveryPlugin; diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java index cd6eb5357ea97..226aec437fc0f 100644 --- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java @@ -295,7 +295,7 @@ private void writeBlobResumable(BlobInfo blobInfo, InputStream inputStream, long * It is not enough to wrap the call to Streams#copy, we have to wrap the privileged calls too; this is because Streams#copy * is in the stacktrace and is not granted the permissions needed to close and write the channel. */ - org.opensearch.core.internal.io.Streams.copy(inputStream, Channels.newOutputStream(new WritableByteChannel() { + org.opensearch.common.util.io.Streams.copy(inputStream, Channels.newOutputStream(new WritableByteChannel() { @SuppressForbidden(reason = "channel is based on a socket") @Override diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java index 5448799e7f81b..e15b37f209c5f 100644 --- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java @@ -44,7 +44,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.SpecialPermission; import org.opensearch.common.SuppressForbidden; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.FilterInputStream; import java.io.IOException; diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java index f751d63232f79..477b5ab9cceeb 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java @@ -40,7 +40,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.io.InputStream; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java index b847890f1ea89..e08a19ede0380 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -51,7 +51,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.CountDown; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.blobstore.AbstractBlobContainerRetriesTestCase; import org.junit.After; import org.junit.Before; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java index 1427429f0b766..9c9179c584a38 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java @@ -39,7 +39,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.nio.BytesWriteHandler; import org.opensearch.nio.InboundChannelBuffer; import org.opensearch.nio.Page; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java index 1229aab244a6d..edaee15507df9 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java @@ -53,7 +53,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.nio.BytesChannelContext; import org.opensearch.nio.ChannelFactory; import org.opensearch.nio.Config; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java index 230f89bbafe9f..b42fb7fc4a749 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java @@ -41,8 +41,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.core.internal.io.IOUtils; -import org.opensearch.core.internal.net.NetUtils; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.common.util.net.NetUtils; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.transport.MockTransportService; import org.opensearch.test.transport.StubbableTransport; diff --git a/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java index ec6789d702743..e54a3a1221fc9 100644 --- a/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java +++ b/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java @@ -58,7 +58,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.query.InnerHitBuilder; import org.opensearch.index.query.MatchQueryBuilder; import org.opensearch.index.query.QueryBuilders; diff --git a/qa/os/src/test/java/org/opensearch/packaging/test/PackagingTestCase.java b/qa/os/src/test/java/org/opensearch/packaging/test/PackagingTestCase.java index 259ae6e766c8e..7551bddc3226c 100644 --- a/qa/os/src/test/java/org/opensearch/packaging/test/PackagingTestCase.java +++ b/qa/os/src/test/java/org/opensearch/packaging/test/PackagingTestCase.java @@ -42,7 +42,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.CheckedConsumer; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.packaging.util.Archives; import org.opensearch.packaging.util.Distribution; import org.opensearch.packaging.util.Docker; diff --git a/qa/os/src/test/java/org/opensearch/packaging/util/FileUtils.java b/qa/os/src/test/java/org/opensearch/packaging/util/FileUtils.java index d7005a17926ad..5169ce18fff79 100644 --- a/qa/os/src/test/java/org/opensearch/packaging/util/FileUtils.java +++ b/qa/os/src/test/java/org/opensearch/packaging/util/FileUtils.java @@ -33,7 +33,7 @@ package org.opensearch.packaging.util; import org.apache.logging.log4j.Logger; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.hamcrest.FeatureMatcher; import org.hamcrest.Matcher; diff --git a/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java b/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java index 5f73144501f94..cc6e286d812ce 100644 --- a/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java +++ b/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java @@ -39,7 +39,7 @@ import org.opensearch.client.RestClient; import org.opensearch.client.RestHighLevelClient; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.rest.OpenSearchRestTestCase; import java.io.IOException; diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml new file mode 100644 index 0000000000000..321883a1063e7 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml @@ -0,0 +1,71 @@ +"search across indices with mixed long and double numeric types": + - skip: + version: " - 2.6.99" + reason: relies on numeric sort optimization that landed in 2.7.0 only + + - do: + indices.create: + index: test_1 + body: + settings: + number_of_shards: "1" + number_of_replicas: "0" + mappings: + properties: + counter: + type: long + + - do: + indices.create: + index: test_2 + body: + settings: + number_of_shards: "1" + number_of_replicas: "0" + mappings: + properties: + counter: + type: double + - do: + bulk: + refresh: true + body: + - index: + _index: test_1 + - counter: 223372036854775800 + - index: + _index: test_2 + - counter: 1223372036854775800.23 + - index: + _index: test_2 + - counter: 184.4 + + - do: + search: + index: test_* + rest_total_hits_as_int: true + body: + sort: [{ counter: desc }] + - match: { hits.total: 3 } + - length: { hits.hits: 3 } + - match: { hits.hits.0._index: test_2 } + - match: { hits.hits.0._source.counter: 1223372036854775800.23 } + - match: { hits.hits.0.sort: [1223372036854775800.23] } + - match: { hits.hits.1._index: test_1 } + - match: { hits.hits.1._source.counter: 223372036854775800 } + - match: { hits.hits.1.sort: [223372036854775800] } + + - do: + search: + index: test_* + rest_total_hits_as_int: true + body: + sort: [{ counter: asc }] + - match: { hits.total: 3 } + - length: { hits.hits: 3 } + - match: { hits.hits.0._index: test_2 } + - match: { hits.hits.0._source.counter: 184.4 } + - match: { hits.hits.0.sort: [184.4] } + - match: { hits.hits.1._index: test_1 } + - match: { hits.hits.1._source.counter: 223372036854775800 } + - match: { hits.hits.1.sort: [223372036854775800] } diff --git a/sandbox/plugins/concurrent-search/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java b/sandbox/plugins/concurrent-search/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java index 51cb3c8c0cddc..0b88abb314fcd 100644 --- a/sandbox/plugins/concurrent-search/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java +++ b/sandbox/plugins/concurrent-search/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java @@ -36,7 +36,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.search.RandomApproximationQuery; import org.apache.lucene.tests.util.TestUtil; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.search.internal.ContextIndexSearcher; import org.opensearch.search.profile.ProfileResult; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/licenses/lucene-analysis-common-9.5.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.5.0.jar.sha1 deleted file mode 100644 index bcd78036b0c99..0000000000000 --- a/server/licenses/lucene-analysis-common-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f68660102455a466f98cac0501723bed7e7c6407 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-analysis-common-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..482f06f6e7124 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +af24672cb028edc12d3fb1d5cf07aff557042539 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1 deleted file mode 100644 index 4000ee7fc0e93..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -66be22239d5058b0b8a8aeba03dc047a276efafa \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-backward-codecs-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..0f0e16a81c439 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +7067e30e4374edc78ec3976787b6f1b9339c70ba \ No newline at end of file diff --git a/server/licenses/lucene-core-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-core-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..6022e4d89ac5e --- /dev/null +++ b/server/licenses/lucene-core-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +b72735dd63d0bd9c35534e8106d19e4df8738246 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.5.0.jar.sha1 b/server/licenses/lucene-grouping-9.5.0.jar.sha1 deleted file mode 100644 index 08ed52d3379ec..0000000000000 --- a/server/licenses/lucene-grouping-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b15fe0a55a82168c810d0447ec2e244d16d94f01 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-grouping-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..bd41301b87868 --- /dev/null +++ b/server/licenses/lucene-grouping-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +e20605f667703c8c0f11c42a39cd989b4449ba07 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.5.0.jar.sha1 b/server/licenses/lucene-highlighter-9.5.0.jar.sha1 deleted file mode 100644 index 0df6615e43c66..0000000000000 --- a/server/licenses/lucene-highlighter-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8b5e3cea3370838bda1d730cf55176c24a763d2e \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-highlighter-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..47a5fe2d2e5e4 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +aeb021e2a05353b75903da7f0d2c887e467d7050 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.5.0.jar.sha1 b/server/licenses/lucene-join-9.5.0.jar.sha1 deleted file mode 100644 index d441e3215820a..0000000000000 --- a/server/licenses/lucene-join-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -85a4208614a5660297effce441883687b010073b \ No newline at end of file diff --git a/server/licenses/lucene-join-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-join-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..ab5023e4b0380 --- /dev/null +++ b/server/licenses/lucene-join-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +0f15970748550dfe1b030a493e9c6f195c7b8cbf \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.5.0.jar.sha1 b/server/licenses/lucene-memory-9.5.0.jar.sha1 deleted file mode 100644 index 0fb289af21987..0000000000000 --- a/server/licenses/lucene-memory-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d632e63d08837be715046c8ccb4fb804acd3d7e4 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-memory-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..13f80932533ef --- /dev/null +++ b/server/licenses/lucene-memory-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +714708d7e5566ea2e506d8df87066186befbb59f \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.5.0.jar.sha1 b/server/licenses/lucene-misc-9.5.0.jar.sha1 deleted file mode 100644 index 985fbb18ea6d0..0000000000000 --- a/server/licenses/lucene-misc-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -64773801b8ba8141f4256d22da598de40d6f3033 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-misc-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..261fc772de397 --- /dev/null +++ b/server/licenses/lucene-misc-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +d5e017e7600bb6b04e3610274f1d115774e9c212 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.5.0.jar.sha1 b/server/licenses/lucene-queries-9.5.0.jar.sha1 deleted file mode 100644 index 4ac6d8161fc58..0000000000000 --- a/server/licenses/lucene-queries-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ae7930fa0ea91198905d695a222ef7f09de1c2dd \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-queries-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..9b9e237ef3d41 --- /dev/null +++ b/server/licenses/lucene-queries-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +8ff649c6e6c706ece75d66db711448cd858def22 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.5.0.jar.sha1 b/server/licenses/lucene-queryparser-9.5.0.jar.sha1 deleted file mode 100644 index 45b6229e713c2..0000000000000 --- a/server/licenses/lucene-queryparser-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b3e3d9434bcfce242e242364c64eab0888d285e2 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-queryparser-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..a57f175234217 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +d9c88b97f94efdae9950cc00f3e2927ee1e6290e \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.5.0.jar.sha1 b/server/licenses/lucene-sandbox-9.5.0.jar.sha1 deleted file mode 100644 index d43ebd2d27119..0000000000000 --- a/server/licenses/lucene-sandbox-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -408be287b0f421cf8afd655579a0cc65aba61b28 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-sandbox-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..a2aae40994e9a --- /dev/null +++ b/server/licenses/lucene-sandbox-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +4c0094dc0d9b9e4affa6fd19572c2b02ad017bf0 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1 deleted file mode 100644 index b44f49f3601ba..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4960c09365d20f024bf668f60d7c3b4e54f03750 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-spatial-extras-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..848fce19db14c --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +3a715552f9d21333766a4c2fbb6e7a202e79aac2 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.5.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.5.0.jar.sha1 deleted file mode 100644 index 369e569ee754f..0000000000000 --- a/server/licenses/lucene-spatial3d-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -01ed55dd384e654076b76d7083feee6ac3efe02b \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-spatial3d-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..1ad8385976cc8 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +95cc1bd66ae35122828dd6a0a29f0747c117d51e \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.5.0.jar.sha1 b/server/licenses/lucene-suggest-9.5.0.jar.sha1 deleted file mode 100644 index c44ac0ec028f4..0000000000000 --- a/server/licenses/lucene-suggest-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -195624eb3a09b3dc37cc5fa51ae2f7bd61553985 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.6.0-snapshot-a3ae27f.jar.sha1 b/server/licenses/lucene-suggest-9.6.0-snapshot-a3ae27f.jar.sha1 new file mode 100644 index 0000000000000..2fda6fdae3e09 --- /dev/null +++ b/server/licenses/lucene-suggest-9.6.0-snapshot-a3ae27f.jar.sha1 @@ -0,0 +1 @@ +bc7faa1fbb9c8b3e5dd975942a7c85c82cbc4c3d \ No newline at end of file diff --git a/server/src/internalClusterTest/java/org/opensearch/action/RejectionActionIT.java b/server/src/internalClusterTest/java/org/opensearch/action/RejectionActionIT.java index f930b9e9cfda0..bda24b48b7f10 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/RejectionActionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/RejectionActionIT.java @@ -38,7 +38,7 @@ import org.opensearch.action.search.SearchType; import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.query.QueryBuilders; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkRejectionIT.java b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkRejectionIT.java index aa2f2a214de27..7dde67791a2ec 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkRejectionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkRejectionIT.java @@ -35,7 +35,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.WriteRequest; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.IndexService; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java index ee2a8784fa0ed..b0aeb786b0268 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java @@ -61,7 +61,7 @@ import org.opensearch.common.io.FileSystemUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.Index; import org.opensearch.index.shard.ShardId; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java index 8059266916f3f..5d6bed4629587 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java @@ -47,7 +47,7 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.indices.ShardLimitValidator; import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.snapshots.SnapshotState; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/IndexingPressureIT.java b/server/src/internalClusterTest/java/org/opensearch/index/IndexingPressureIT.java index b461322735ce0..b63ffb2653741 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/IndexingPressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/IndexingPressureIT.java @@ -46,7 +46,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.InternalSettingsPlugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java b/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java index 35d6a9ef0ef1d..bcb4a199bc32d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java @@ -14,7 +14,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; import org.opensearch.indices.replication.SegmentReplicationBaseIT; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureIT.java b/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureIT.java index f14540cec4e2b..0c95b71be8b91 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureIT.java @@ -22,7 +22,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureSettingsIT.java index 1cdf8e702aafa..dafc8fd16e892 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/ShardIndexingPressureSettingsIT.java @@ -24,7 +24,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index 9cdac985310bd..11f187ac6e619 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -62,7 +62,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLock; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java index ad4789981fa42..e448b018dd061 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java @@ -80,7 +80,7 @@ import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.xcontent.XContentType; import org.opensearch.gateway.ReplicaShardAllocatorIT; import org.opensearch.index.Index; diff --git a/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java index 75deb731881ec..c09bd743ca361 100644 --- a/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java @@ -39,7 +39,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.blobstore.OpenSearchBlobStoreRepositoryIntegTestCase; import java.io.IOException; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java index 1482e73efdace..197a364ddeaa0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java @@ -49,6 +49,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.Future; @@ -513,6 +514,292 @@ public void testShardRoutingWithNetworkDisruption_FailOpenEnabled() throws Excep assertNoSearchInAZ("a"); } + public void testStrictWeightedRoutingWithCustomString_FailOpenEnabled() throws Exception { + Settings commonSettings = Settings.builder() + .put("cluster.routing.allocation.awareness.attributes", "zone") + .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c") + .put("cluster.routing.weighted.fail_open", true) + .put("cluster.routing.weighted.strict", true) + .build(); + + int nodeCountPerAZ = 1; + Map> nodeMap = setupCluster(nodeCountPerAZ, commonSettings); + + int numShards = 10; + int numReplicas = 1; + setUpIndexing(numShards, numReplicas); + + logger.info("--> creating network partition disruption"); + final String clusterManagerNode1 = internalCluster().getClusterManagerName(); + Set nodesInOneSide = Stream.of(clusterManagerNode1, nodeMap.get("b").get(0)).collect(Collectors.toCollection(HashSet::new)); + Set nodesInOtherSide = Stream.of(nodeMap.get("a").get(0)).collect(Collectors.toCollection(HashSet::new)); + + logger.info("--> setting shard routing weights for weighted round robin"); + Map weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0); + setShardRoutingWeights(weights); + + NetworkDisruption networkDisruption = new NetworkDisruption( + new NetworkDisruption.TwoPartitions(nodesInOneSide, nodesInOtherSide), + NetworkDisruption.UNRESPONSIVE + ); + internalCluster().setDisruptionScheme(networkDisruption); + + logger.info("--> network disruption is started"); + networkDisruption.startDisrupting(); + + Set hitNodes = new HashSet<>(); + Future[] responses = new Future[50]; + String customPreference = randomAlphaOfLength(10); + logger.info("--> making search requests"); + for (int i = 0; i < 50; i++) { + responses[i] = internalCluster().client(nodeMap.get("b").get(0)) + .prepareSearch("test") + .setPreference(customPreference) + .setSize(100) + .setQuery(QueryBuilders.matchAllQuery()) + .execute(); + } + + logger.info("--> network disruption is stopped"); + networkDisruption.stopDisrupting(); + + logger.info("--> shards should fail due to network disruption"); + for (int i = 0; i < 50; i++) { + try { + SearchResponse searchResponse = responses[i].get(); + assertEquals(searchResponse.getFailedShards(), 0); + for (int j = 0; j < searchResponse.getHits().getHits().length; j++) { + hitNodes.add(searchResponse.getHits().getAt(j).getShard().getNodeId()); + } + } catch (Exception t) { + fail("search should not fail"); + } + } + + try { + assertSearchInAZ("b"); + } catch (AssertionError ae) { + assertSearchInAZ("c"); + } + assertNoSearchInAZ("a"); + } + + public void testStrictWeightedRoutingWithCustomString_FailOpenDisabled() throws Exception { + Settings commonSettings = Settings.builder() + .put("cluster.routing.allocation.awareness.attributes", "zone") + .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c") + .put("cluster.routing.weighted.fail_open", false) + .put("cluster.routing.weighted.strict", true) + .build(); + + int nodeCountPerAZ = 1; + Map> nodeMap = setupCluster(nodeCountPerAZ, commonSettings); + + int numShards = 10; + int numReplicas = 1; + setUpIndexing(numShards, numReplicas); + + logger.info("--> creating network partition disruption"); + final String clusterManagerNode1 = internalCluster().getClusterManagerName(); + Set nodesInOneSide = Stream.of(clusterManagerNode1, nodeMap.get("b").get(0)).collect(Collectors.toCollection(HashSet::new)); + Set nodesInOtherSide = Stream.of(nodeMap.get("a").get(0)).collect(Collectors.toCollection(HashSet::new)); + + logger.info("--> setting shard routing weights for weighted round robin"); + Map weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0); + setShardRoutingWeights(weights); + + NetworkDisruption networkDisruption = new NetworkDisruption( + new NetworkDisruption.TwoPartitions(nodesInOneSide, nodesInOtherSide), + NetworkDisruption.UNRESPONSIVE + ); + internalCluster().setDisruptionScheme(networkDisruption); + + logger.info("--> network disruption is started"); + networkDisruption.startDisrupting(); + + Set hitNodes = new HashSet<>(); + Future[] responses = new Future[50]; + String customPreference = randomAlphaOfLength(10); + logger.info("--> making search requests"); + for (int i = 0; i < 50; i++) { + responses[i] = internalCluster().client(nodeMap.get("b").get(0)) + .prepareSearch("test") + .setPreference(customPreference) + .setSize(100) + .setQuery(QueryBuilders.matchAllQuery()) + .execute(); + } + + logger.info("--> network disruption is stopped"); + networkDisruption.stopDisrupting(); + + logger.info("--> shards should fail due to network disruption"); + for (int i = 0; i < 50; i++) { + try { + SearchResponse searchResponse = responses[i].get(); + assertNotEquals(searchResponse.getFailedShards(), 0); + for (int j = 0; j < searchResponse.getHits().getHits().length; j++) { + hitNodes.add(searchResponse.getHits().getAt(j).getShard().getNodeId()); + } + } catch (Exception t) { + fail("search should not fail"); + } + } + + DiscoveryNodes dataNodes = internalCluster().clusterService().state().nodes(); + Set expectedHotNodes = new HashSet<>(); + for (DiscoveryNode node : dataNodes) { + if (node.getAttributes().getOrDefault("zone", "").equals("b")) { + expectedHotNodes.add(node.getId()); + } + } + + assertEquals(expectedHotNodes, hitNodes); + + assertSearchInAZ("b"); + assertNoSearchInAZ("c"); + assertNoSearchInAZ("a"); + } + + /** + * Should failopen shards even if failopen enabled with custom search preference. + * @throws Exception + */ + public void testStrictWeightedRoutingWithShardPrefNetworkDisruption_FailOpenEnabled() throws Exception { + Settings commonSettings = Settings.builder() + .put("cluster.routing.allocation.awareness.attributes", "zone") + .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c") + .put("cluster.routing.weighted.fail_open", true) + .put("cluster.routing.weighted.strict", true) + .build(); + + int nodeCountPerAZ = 1; + Map> nodeMap = setupCluster(nodeCountPerAZ, commonSettings); + + int numShards = 10; + int numReplicas = 1; + setUpIndexing(numShards, numReplicas); + + logger.info("--> creating network partition disruption"); + final String clusterManagerNode1 = internalCluster().getClusterManagerName(); + Set nodesInOneSide = Stream.of(clusterManagerNode1, nodeMap.get("c").get(0)).collect(Collectors.toCollection(HashSet::new)); + Set nodesInOtherSide = Stream.of(nodeMap.get("a").get(0)).collect(Collectors.toCollection(HashSet::new)); + + logger.info("--> setting shard routing weights for weighted round robin"); + Map weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0); + setShardRoutingWeights(weights); + + NetworkDisruption networkDisruption = new NetworkDisruption( + new NetworkDisruption.TwoPartitions(nodesInOneSide, nodesInOtherSide), + NetworkDisruption.UNRESPONSIVE + ); + internalCluster().setDisruptionScheme(networkDisruption); + + logger.info("--> network disruption is started"); + networkDisruption.startDisrupting(); + + Future[] responses = new Future[50]; + DiscoveryNodes dataNodes = internalCluster().clusterService().state().nodes(); + ShardId shardId = internalCluster().clusterService() + .state() + .getRoutingTable() + .index("test") + .randomAllActiveShardsIt() + .getShardRoutings() + .stream() + .filter(shard -> { + return dataNodes.get(shard.currentNodeId()).getAttributes().getOrDefault("zone", "").equals("c"); + }) + .findFirst() + .get() + .shardId(); + + for (int i = 0; i < 50; i++) { + responses[i] = internalCluster().client(nodeMap.get("c").get(0)) + .prepareSearch("test") + .setPreference(String.format(Locale.ROOT, "_shards:%s", shardId.getId())) + .setSize(100) + .setQuery(QueryBuilders.matchAllQuery()) + .execute(); + } + + logger.info("--> network disruption is stopped"); + networkDisruption.stopDisrupting(); + + for (int i = 0; i < 50; i++) { + try { + SearchResponse searchResponse = responses[i].get(); + assertEquals(searchResponse.getFailedShards(), 0); + } catch (Exception t) { + fail("search should not fail"); + } + } + + assertNoSearchInAZ("a"); + try { + assertSearchInAZ("c"); + } catch (AssertionError ae) { + assertSearchInAZ("b"); + } + } + + public void testStrictWeightedRoutingWithShardPref() throws Exception { + Settings commonSettings = Settings.builder() + .put("cluster.routing.allocation.awareness.attributes", "zone") + .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c") + .put("cluster.routing.weighted.fail_open", true) + .put("cluster.routing.weighted.strict", true) + .build(); + + int nodeCountPerAZ = 1; + Map> nodeMap = setupCluster(nodeCountPerAZ, commonSettings); + + int numShards = 10; + int numReplicas = 1; + setUpIndexing(numShards, numReplicas); + + logger.info("--> setting shard routing weights for weighted round robin"); + Map weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0); + setShardRoutingWeights(weights); + + DiscoveryNodes dataNodes = internalCluster().clusterService().state().nodes(); + ShardId shardId = internalCluster().clusterService() + .state() + .getRoutingTable() + .index("test") + .randomAllActiveShardsIt() + .getShardRoutings() + .stream() + .filter(shard -> { + return dataNodes.get(shard.currentNodeId()).getAttributes().getOrDefault("zone", "").equals("c"); + }) + .findFirst() + .get() + .shardId(); + + Future[] responses = new Future[50]; + logger.info("--> making search requests"); + for (int i = 0; i < 50; i++) { + responses[i] = internalCluster().client(nodeMap.get("b").get(0)) + .prepareSearch("test") + .setPreference(String.format(Locale.ROOT, "_shards:%s", shardId.getId())) + .setSize(100) + .setQuery(QueryBuilders.matchAllQuery()) + .execute(); + } + + for (int i = 0; i < 50; i++) { + try { + SearchResponse searchResponse = responses[i].get(); + assertEquals(searchResponse.getFailedShards(), 0); + assertNotEquals(searchResponse.getHits().getTotalHits().value, 0); + } catch (Exception t) { + fail("search should not fail"); + } + } + assertNoSearchInAZ("c"); + } + private void assertNoSearchInAZ(String az) { ImmutableOpenMap dataNodes = internalCluster().clusterService().state().nodes().getDataNodes(); String dataNodeId = null; @@ -806,7 +1093,6 @@ public void testMultiGetWithNetworkDisruption_FailOpenDisabled() throws Exceptio * Assert that preference search with custom string doesn't hit a node in weighed away az */ public void testStrictWeightedRoutingWithCustomString() { - Settings commonSettings = Settings.builder() .put("cluster.routing.allocation.awareness.attributes", "zone") .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c") @@ -844,13 +1130,18 @@ public void testStrictWeightedRoutingWithCustomString() { .get(); // make search requests with custom string - searchResponse = internalCluster().client(nodeMap.get("a").get(0)) + internalCluster().client(nodeMap.get("a").get(0)) .prepareSearch() .setSize(20) .setPreference(customPreference) + .setQuery(QueryBuilders.matchAllQuery()) .get(); // assert search on data nodes on az c (weighed away az) - assertSearchInAZ("c"); + try { + assertSearchInAZ("c"); + } catch (AssertionError ae) { + assertSearchInAZ("a"); + } } @@ -889,24 +1180,61 @@ public void testPreferenceSearchWithWeightedRouting() { SearchResponse searchResponse = internalCluster().client(nodeMap.get("b").get(0)) .prepareSearch() - .setSize(0) - .setPreference("_local") + .setPreference(randomFrom("_local", "_prefer_nodes:" + "zone:a", customPreference)) .get(); assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus()); - searchResponse = internalCluster().client(nodeMap.get("b").get(0)) + searchResponse = internalCluster().client(nodeMap.get("a").get(0)) .prepareSearch() - .setSize(0) .setPreference( "_only_nodes:" + nodeIDMap.get(nodeInZoneA) + "," + nodeIDMap.get(nodeInZoneB) + "," + nodeIDMap.get(nodeInZoneC) ) .get(); assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus()); + } - searchResponse = internalCluster().client(nodeMap.get("b").get(0)) + public void testPreferenceSearchWithIgnoreWeightedRouting() { + Settings commonSettings = Settings.builder() + .put("cluster.routing.allocation.awareness.attributes", "zone") + .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c") + .put("cluster.routing.weighted.fail_open", true) + .put("cluster.routing.weighted.strict", false) + .put("cluster.routing.ignore_weighted_routing", true) + .build(); + + int nodeCountPerAZ = 1; + Map> nodeMap = setupCluster(nodeCountPerAZ, commonSettings); + + int numShards = 10; + int numReplicas = 2; + setUpIndexing(numShards, numReplicas); + + logger.info("--> setting shard routing weights for weighted round robin"); + Map weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0); + setShardRoutingWeights(weights); + + String customPreference = randomAlphaOfLength(10); + String nodeInZoneA = nodeMap.get("a").get(0); + String nodeInZoneB = nodeMap.get("b").get(0); + String nodeInZoneC = nodeMap.get("c").get(0); + + Map nodeIDMap = new HashMap<>(); + DiscoveryNodes dataNodes = internalCluster().clusterService().state().nodes(); + for (DiscoveryNode node : dataNodes) { + nodeIDMap.put(node.getName(), node.getId()); + } + + SearchResponse searchResponse = internalCluster().client(nodeMap.get("b").get(0)) .prepareSearch() - .setSize(0) - .setPreference("_prefer_nodes:zone:a") + .setPreference(randomFrom("_local", "_prefer_nodes:" + "zone:a", customPreference)) + .get(); + assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus()); + + searchResponse = internalCluster().client(nodeMap.get("a").get(0)) + .prepareSearch() + .setPreference( + "_only_nodes:" + nodeIDMap.get(nodeInZoneA) + "," + nodeIDMap.get(nodeInZoneB) + "," + nodeIDMap.get(nodeInZoneC) + ) .get(); assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus()); } @@ -915,7 +1243,6 @@ public void testPreferenceSearchWithWeightedRouting() { * Assert that preference based search with preference type is not allowed with strict weighted shard routing */ public void testStrictWeightedRouting() { - Settings commonSettings = Settings.builder() .put("cluster.routing.allocation.awareness.attributes", "zone") .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c") @@ -935,11 +1262,6 @@ public void testStrictWeightedRouting() { setShardRoutingWeights(weights); String nodeInZoneA = nodeMap.get("a").get(0); - assertThrows( - PreferenceBasedSearchNotAllowedException.class, - () -> internalCluster().client(nodeMap.get("b").get(0)).prepareSearch().setSize(0).setPreference("_local").get() - ); - assertThrows( PreferenceBasedSearchNotAllowedException.class, () -> internalCluster().client(nodeMap.get("b").get(0)) @@ -957,7 +1279,48 @@ public void testStrictWeightedRouting() { .setPreference("_prefer_nodes:" + nodeInZoneA) .get() ); + } + public void testStrictWeightedRoutingAllowedForSomeSearchPrefs() { + Settings commonSettings = Settings.builder() + .put("cluster.routing.allocation.awareness.attributes", "zone") + .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c") + .put("cluster.routing.weighted.fail_open", true) + .put("cluster.routing.weighted.strict", true) + .build(); + + int nodeCountPerAZ = 1; + Map> nodeMap = setupCluster(nodeCountPerAZ, commonSettings); + + int numShards = 10; + int numReplicas = 1; + setUpIndexing(numShards, numReplicas); + + logger.info("--> setting shard routing weights for weighted round robin"); + Map weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0); + setShardRoutingWeights(weights); + String nodeInZoneA = nodeMap.get("a").get(0); + String customPreference = randomAlphaOfLength(10); + + SearchResponse searchResponse = internalCluster().client(nodeMap.get("b").get(0)) + .prepareSearch() + .setSize(0) + .setPreference("_only_local:" + nodeInZoneA) + .get(); + assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus()); + + searchResponse = internalCluster().client(nodeMap.get("b").get(0)) + .prepareSearch() + .setSize(0) + .setPreference("_local:" + nodeInZoneA) + .get(); + assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus()); + + searchResponse = internalCluster().client(nodeMap.get("b").get(0)).prepareSearch().setSize(0).setPreference("_shards:1").get(); + assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus()); + + searchResponse = internalCluster().client(nodeMap.get("b").get(0)).prepareSearch().setSize(0).setPreference(customPreference).get(); + assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus()); } /** diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java index 277d6b45be4ea..3444ddbca9e65 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java @@ -60,7 +60,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.Streams; +import org.opensearch.common.util.io.Streams; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java index a07b245db2b21..483d698f3c9a4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/MultiClusterRepoAccessIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/MultiClusterRepoAccessIT.java index 13aee8b8e18a6..4aabe96bcbe60 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/MultiClusterRepoAccessIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/MultiClusterRepoAccessIT.java @@ -33,7 +33,7 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.repositories.RepositoryException; import org.opensearch.snapshots.mockstore.MockRepository; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java index 7857c6430a5e3..f4a34ddf847de 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java @@ -35,10 +35,15 @@ import org.opensearch.monitor.fs.FsInfo; import org.opensearch.repositories.fs.FsRepository; +import java.io.IOException; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; @@ -537,33 +542,68 @@ public void testPruneFileCacheOnIndexDeletion() throws Exception { assertAllNodesFileCacheEmpty(); } - public void testCacheFilesAreClosedAfterUse() throws Exception { - final int numReplicasIndex = randomIntBetween(1, 4); + /** + * Test scenario that checks the cache folder location on search nodes for the restored index on snapshot restoration + * and ensures the index folder is cleared on all nodes post index deletion + */ + public void testCacheIndexFilesClearedOnDelete() throws Exception { + final int numReplicas = randomIntBetween(1, 4); + final int numShards = numReplicas + 1; final String indexName = "test-idx"; final String restoredIndexName = indexName + "-copy"; final String repoName = "test-repo"; final String snapshotName = "test-snap"; final Client client = client(); - internalCluster().ensureAtLeastNumSearchAndDataNodes(numReplicasIndex + 1); - createIndexWithDocsAndEnsureGreen(1, 100, indexName); + internalCluster().ensureAtLeastNumSearchAndDataNodes(numShards); + createIndexWithDocsAndEnsureGreen(numReplicas, 100, indexName); createRepositoryWithSettings(null, repoName); takeSnapshot(client, snapshotName, repoName, indexName); restoreSnapshotAndEnsureGreen(client, snapshotName, repoName); assertDocCount(restoredIndexName, 100L); + // The index count will be 1 since there is only a single restored index "test-idx-copy" + assertCacheDirectoryReplicaAndIndexCount(numShards, 1); + // The local cache files should be closed by deleting the restored index deleteIndicesAndEnsureGreen(client, restoredIndexName); logger.info("--> validate cache file path is deleted"); - // Get path of cache files - final NodeEnvironment nodeEnv = internalCluster().getInstance(NodeEnvironment.class); - Path fileCachePath = nodeEnv.fileCacheNodePath().fileCachePath; + // The index count will be 0 since the only restored index "test-idx-copy" was deleted + assertCacheDirectoryReplicaAndIndexCount(numShards, 0); + logger.info("--> validated that the cache file path doesn't exist"); + } - if (Files.exists(fileCachePath)) { - fail("Cache file path isn't deleted."); - } else { - logger.info("--> validated that the cache file path doesn't exist"); + /** + * Asserts the cache folder count to match the number of shards and the number of indices within the cache folder + * as provided. + * @param numCacheFolderCount total number of cache folders that should exist for the test case + * @param numIndexCount total number of index folder locations that should exist within the cache folder + */ + private void assertCacheDirectoryReplicaAndIndexCount(int numCacheFolderCount, int numIndexCount) throws IOException { + // Get the available NodeEnvironment instances + Iterable nodeEnvironments = internalCluster().getInstances(NodeEnvironment.class); + + // Filter out search NodeEnvironment(s) since FileCache is initialized only on search nodes and + // collect the path for all the cache locations on search nodes. + List searchNodeFileCachePaths = StreamSupport.stream(nodeEnvironments.spliterator(), false) + .filter(nodeEnv -> nodeEnv.fileCache() != null) + .map(nodeEnv -> nodeEnv.fileCacheNodePath().fileCachePath) + .collect(Collectors.toList()); + + // Walk through the cache directory on nodes + for (Path fileCachePath : searchNodeFileCachePaths) { + assertTrue(Files.exists(fileCachePath)); + assertTrue(Files.isDirectory(fileCachePath)); + try (DirectoryStream cachePathStream = Files.newDirectoryStream(fileCachePath)) { + Path nodeLockIdPath = cachePathStream.iterator().next(); + assertTrue(Files.isDirectory(nodeLockIdPath)); + try (Stream dataPathStream = Files.list(nodeLockIdPath)) { + assertEquals(numIndexCount, dataPathStream.count()); + } + } } + // Verifies if all the shards (primary and replica) have been deleted + assertEquals(numCacheFolderCount, searchNodeFileCachePaths.size()); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index b6fa4fbc2fc96..69b947588bea0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -49,7 +49,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/ExceptionsHelper.java b/server/src/main/java/org/opensearch/ExceptionsHelper.java index aa638514f9cc7..486fb7a3c72ee 100644 --- a/server/src/main/java/org/opensearch/ExceptionsHelper.java +++ b/server/src/main/java/org/opensearch/ExceptionsHelper.java @@ -43,7 +43,7 @@ import org.opensearch.common.CheckedSupplier; import org.opensearch.common.Nullable; import org.opensearch.common.compress.NotXContentException; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.Index; import org.opensearch.rest.RestStatus; diff --git a/server/src/main/java/org/opensearch/LegacyESVersion.java b/server/src/main/java/org/opensearch/LegacyESVersion.java index e9333ffb58148..6dca66b181e76 100644 --- a/server/src/main/java/org/opensearch/LegacyESVersion.java +++ b/server/src/main/java/org/opensearch/LegacyESVersion.java @@ -33,10 +33,11 @@ package org.opensearch; import org.opensearch.common.Strings; -import org.opensearch.common.collect.ImmutableOpenIntMap; -import org.opensearch.common.collect.ImmutableOpenMap; +import org.opensearch.core.Assertions; import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; /** * The Contents of this file were originally moved from {@link Version}. @@ -53,11 +54,11 @@ public class LegacyESVersion extends Version { public static final LegacyESVersion V_7_2_0 = new LegacyESVersion(7020099, org.apache.lucene.util.Version.LUCENE_8_0_0); // todo move back to Version.java if retiring legacy version support - protected static final ImmutableOpenIntMap idToVersion; - protected static final ImmutableOpenMap stringToVersion; + protected static final Map idToVersion; + protected static final Map stringToVersion; static { - final ImmutableOpenIntMap.Builder builder = ImmutableOpenIntMap.builder(); - final ImmutableOpenMap.Builder builderByString = ImmutableOpenMap.builder(); + final Map builder = new HashMap<>(); + final Map builderByString = new HashMap<>(); for (final Field declaredField : LegacyESVersion.class.getFields()) { if (declaredField.getType().equals(Version.class) || declaredField.getType().equals(LegacyESVersion.class)) { @@ -119,8 +120,8 @@ public class LegacyESVersion extends Version { builder.put(V_EMPTY_ID, V_EMPTY); builderByString.put(V_EMPTY.toString(), V_EMPTY); - idToVersion = builder.build(); - stringToVersion = builderByString.build(); + idToVersion = Map.copyOf(builder); + stringToVersion = Map.copyOf(builderByString); } protected LegacyESVersion(int id, org.apache.lucene.util.Version luceneVersion) { diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java index 29aafc76262bb..16cf1ebcc173c 100644 --- a/server/src/main/java/org/opensearch/Version.java +++ b/server/src/main/java/org/opensearch/Version.java @@ -92,7 +92,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_6_0 = new Version(2060099, org.apache.lucene.util.Version.LUCENE_9_5_0); public static final Version V_2_6_1 = new Version(2060199, org.apache.lucene.util.Version.LUCENE_9_5_0); public static final Version V_2_7_0 = new Version(2070099, org.apache.lucene.util.Version.LUCENE_9_5_0); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_5_0); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version CURRENT = V_3_0_0; public static Version readVersion(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/analyze/TransportAnalyzeAction.java index e5efe9055592d..2700d38686833 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -51,7 +51,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java index 862d63aeedf53..ebba7c93af023 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SparseFixedBitSet; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.OpenSearchParseException; import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceAlreadyExistsException; diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java index 3f4f7c2b92512..6199af11a565f 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java @@ -615,14 +615,13 @@ private static Sort createSort(TopFieldDocs[] topFieldDocs) { final SortField[] newFields = new SortField[firstTopDocFields.length]; for (int i = 0; i < firstTopDocFields.length; i++) { - if (firstTopDocFields[i] instanceof SortedNumericSortField && isSortWideningRequired(topFieldDocs, i)) { - final SortedNumericSortField delegate = (SortedNumericSortField) firstTopDocFields[i]; - newFields[i] = new SortedWiderNumericSortField( - delegate.getField(), - delegate.getNumericType(), - delegate.getReverse(), - delegate.getSelector() - ); + final SortField delegate = firstTopDocFields[i]; + final SortField.Type type = delegate instanceof SortedNumericSortField + ? ((SortedNumericSortField) delegate).getNumericType() + : delegate.getType(); + + if (SortedWiderNumericSortField.isTypeSupported(type) && isSortWideningRequired(topFieldDocs, i)) { + newFields[i] = new SortedWiderNumericSortField(delegate.getField(), type, delegate.getReverse()); } else { newFields[i] = firstTopDocFields[i]; } @@ -636,7 +635,7 @@ private static Sort createSort(TopFieldDocs[] topFieldDocs) { */ private static boolean isSortWideningRequired(TopFieldDocs[] topFieldDocs, int sortFieldindex) { for (int i = 0; i < topFieldDocs.length - 1; i++) { - if (topFieldDocs[i].fields[sortFieldindex] != topFieldDocs[i + 1].fields[sortFieldindex]) { + if (!topFieldDocs[i].fields[sortFieldindex].equals(topFieldDocs[i + 1].fields[sortFieldindex])) { return true; } } diff --git a/server/src/main/java/org/opensearch/action/support/RetryableAction.java b/server/src/main/java/org/opensearch/action/support/RetryableAction.java index 281cf728fb18c..cf9c140a54bc8 100644 --- a/server/src/main/java/org/opensearch/action/support/RetryableAction.java +++ b/server/src/main/java/org/opensearch/action/support/RetryableAction.java @@ -38,7 +38,7 @@ import org.opensearch.action.ActionRunnable; import org.opensearch.action.bulk.BackoffPolicy; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java index 944729df2ab1e..423fbd12a6447 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java +++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.store.AlreadyClosedException; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; @@ -50,7 +50,7 @@ import org.opensearch.common.breaker.CircuitBreakingException; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.ReplicationGroup; import org.opensearch.index.shard.ShardId; diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java index e804aa31adb4e..5b4b6a35771b0 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.store.AlreadyClosedException; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index 8c4a550a9b5f5..25b6c002afc47 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -56,7 +56,7 @@ import org.opensearch.common.settings.SecureString; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.BoundTransportAddress; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.monitor.os.OsProbe; diff --git a/server/src/main/java/org/opensearch/bootstrap/Spawner.java b/server/src/main/java/org/opensearch/bootstrap/Spawner.java index e2df60af6c1d1..3a94523966e21 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Spawner.java +++ b/server/src/main/java/org/opensearch/bootstrap/Spawner.java @@ -33,7 +33,7 @@ package org.opensearch.bootstrap; import org.apache.lucene.util.Constants; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.plugins.Platforms; import org.opensearch.plugins.PluginInfo; diff --git a/server/src/main/java/org/opensearch/bootstrap/SystemCallFilter.java b/server/src/main/java/org/opensearch/bootstrap/SystemCallFilter.java index 5792452e84f7a..6347c37a7c7a5 100644 --- a/server/src/main/java/org/opensearch/bootstrap/SystemCallFilter.java +++ b/server/src/main/java/org/opensearch/bootstrap/SystemCallFilter.java @@ -43,7 +43,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.Constants; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java index 5d0ba2c67e961..052c320e9b268 100644 --- a/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java @@ -58,7 +58,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.store.StoreStats; import org.opensearch.monitor.fs.FsInfo; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 8ef8124799bf0..77f3b89f950ce 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -53,7 +53,7 @@ import org.opensearch.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.BytesTransportRequest; import org.opensearch.transport.TransportChannel; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 56f9d9df95d02..f96eb9955593e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -36,7 +36,7 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.action.admin.indices.rollover.RolloverInfo; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java index deb4dfd2581bf..665bdf0c4e2ff 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java @@ -52,7 +52,7 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.inject.Inject; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.DocumentMapper; diff --git a/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java b/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java index dbef876c9a258..72c189f20eaf6 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java @@ -20,6 +20,8 @@ import java.util.List; +import static org.opensearch.cluster.routing.OperationRouting.IGNORE_WEIGHTED_SHARD_ROUTING; + /** * This class contains logic to find next shard to retry search request in case of failure from other shard copy. * This decides if retryable shard search requests can be tried on shard copies present in data @@ -72,9 +74,13 @@ public SearchShardTarget findNext( Runnable onShardSkipped ) { SearchShardTarget next = shardIt.nextOrNull(); + if (ignoreWeightedRouting(clusterState)) { + return next; + } + while (next != null && WeightedRoutingUtils.isWeighedAway(next.getNodeId(), clusterState)) { SearchShardTarget nextShard = next; - if (canFailOpen(nextShard.getShardId(), exception, clusterState)) { + if (canFailOpen(nextShard.getShardId(), shardIt.size(), exception, clusterState)) { logger.info(() -> new ParameterizedMessage("{}: Fail open executed due to exception", nextShard.getShardId()), exception); getWeightedRoutingStats().updateFailOpenCount(); break; @@ -98,10 +104,13 @@ public SearchShardTarget findNext( */ public ShardRouting findNext(final ShardsIterator shardsIt, ClusterState clusterState, Exception exception, Runnable onShardSkipped) { ShardRouting next = shardsIt.nextOrNull(); + if (ignoreWeightedRouting(clusterState)) { + return next; + } while (next != null && WeightedRoutingUtils.isWeighedAway(next.currentNodeId(), clusterState)) { ShardRouting nextShard = next; - if (canFailOpen(nextShard.shardId(), exception, clusterState)) { + if (canFailOpen(nextShard.shardId(), shardsIt.size(), exception, clusterState)) { logger.info(() -> new ParameterizedMessage("{}: Fail open executed due to exception", nextShard.shardId()), exception); getWeightedRoutingStats().updateFailOpenCount(); break; @@ -117,8 +126,8 @@ public ShardRouting findNext(final ShardsIterator shardsIt, ClusterState cluster * @return true if can fail open ie request shard copies present in nodes with weighted shard * routing weight set to zero */ - private boolean canFailOpen(ShardId shardId, Exception exception, ClusterState clusterState) { - return isInternalFailure(exception) || hasInActiveShardCopies(clusterState, shardId); + private boolean canFailOpen(ShardId shardId, int shardItSize, Exception exception, ClusterState clusterState) { + return shardItSize == 1 || isInternalFailure(exception) || hasInActiveShardCopies(clusterState, shardId); } private boolean hasInActiveShardCopies(ClusterState clusterState, ShardId shardId) { @@ -131,6 +140,10 @@ private boolean hasInActiveShardCopies(ClusterState clusterState, ShardId shardI return false; } + private boolean ignoreWeightedRouting(ClusterState clusterState) { + return IGNORE_WEIGHTED_SHARD_ROUTING.get(clusterState.getMetadata().settings()); + } + public WeightedRoutingStats getWeightedRoutingStats() { return WeightedRoutingStats.getInstance(); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 6a877838ece95..711a750ade712 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -324,9 +324,12 @@ public ShardIterator activeInitializingShardsWeightedIt( WeightedRouting weightedRouting, DiscoveryNodes nodes, double defaultWeight, - boolean isFailOpenEnabled + boolean isFailOpenEnabled, + @Nullable Integer seed ) { - final int seed = shufflerForWeightedRouting.nextSeed(); + if (seed == null) { + seed = shufflerForWeightedRouting.nextSeed(); + } List ordered = activeInitializingShardsWithWeights(weightedRouting, nodes, defaultWeight, seed); // append shards for attribute value with weight zero, so that shard search requests can be tried on @@ -350,6 +353,7 @@ public ShardIterator activeInitializingShardsWeightedIt( logger.debug("no shard copies found for shard id [{}] for node attribute with weight zero", shardId); } } + return new PlainShardIterator(shardId, ordered); } @@ -371,21 +375,6 @@ private List activeInitializingShardsWithWeights( return orderedListWithDistinctShards; } - /** - * Returns an iterator over active and initializing shards, shards are ordered by weighted - * round-robin scheduling policy. Uses the passed seed to shuffle the shards. - * - */ - public ShardIterator activeInitializingShardsSimpleWeightedIt( - WeightedRouting weightedRouting, - DiscoveryNodes nodes, - double defaultWeight, - int seed - ) { - List ordered = activeInitializingShardsWithWeights(weightedRouting, nodes, defaultWeight, seed); - return new PlainShardIterator(shardId, ordered); - } - /** * Returns a list containing shard routings ordered using weighted round-robin scheduling. */ diff --git a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java index b247936245151..566eefe2c4f1a 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java @@ -93,16 +93,30 @@ public class OperationRouting { public static final Setting STRICT_WEIGHTED_SHARD_ROUTING_ENABLED = Setting.boolSetting( "cluster.routing.weighted.strict", + true, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + public static final Setting IGNORE_WEIGHTED_SHARD_ROUTING = Setting.boolSetting( + "cluster.routing.ignore_weighted_routing", false, Setting.Property.Dynamic, Setting.Property.NodeScope ); + + private static final List WEIGHTED_ROUTING_RESTRICTED_PREFERENCES = Arrays.asList( + Preference.ONLY_NODES, + Preference.PREFER_NODES + ); + private volatile List awarenessAttributes; private volatile boolean useAdaptiveReplicaSelection; private volatile boolean ignoreAwarenessAttr; private volatile double weightedRoutingDefaultWeight; private volatile boolean isFailOpenEnabled; private volatile boolean isStrictWeightedShardRouting; + private volatile boolean ignoreWeightedRouting; public OperationRouting(Settings settings, ClusterSettings clusterSettings) { // whether to ignore awareness attributes when routing requests @@ -116,11 +130,13 @@ public OperationRouting(Settings settings, ClusterSettings clusterSettings) { this.weightedRoutingDefaultWeight = WEIGHTED_ROUTING_DEFAULT_WEIGHT.get(settings); this.isFailOpenEnabled = WEIGHTED_ROUTING_FAILOPEN_ENABLED.get(settings); this.isStrictWeightedShardRouting = STRICT_WEIGHTED_SHARD_ROUTING_ENABLED.get(settings); + this.ignoreWeightedRouting = IGNORE_WEIGHTED_SHARD_ROUTING.get(settings); clusterSettings.addSettingsUpdateConsumer(USE_ADAPTIVE_REPLICA_SELECTION_SETTING, this::setUseAdaptiveReplicaSelection); clusterSettings.addSettingsUpdateConsumer(IGNORE_AWARENESS_ATTRIBUTES_SETTING, this::setIgnoreAwarenessAttributes); clusterSettings.addSettingsUpdateConsumer(WEIGHTED_ROUTING_DEFAULT_WEIGHT, this::setWeightedRoutingDefaultWeight); clusterSettings.addSettingsUpdateConsumer(WEIGHTED_ROUTING_FAILOPEN_ENABLED, this::setFailOpenEnabled); clusterSettings.addSettingsUpdateConsumer(STRICT_WEIGHTED_SHARD_ROUTING_ENABLED, this::setStrictWeightedShardRouting); + clusterSettings.addSettingsUpdateConsumer(IGNORE_WEIGHTED_SHARD_ROUTING, this::setIgnoreWeightedRouting); } void setUseAdaptiveReplicaSelection(boolean useAdaptiveReplicaSelection) { @@ -143,6 +159,10 @@ void setStrictWeightedShardRouting(boolean strictWeightedShardRouting) { this.isStrictWeightedShardRouting = strictWeightedShardRouting; } + void setIgnoreWeightedRouting(boolean isWeightedRoundRobinEnabled) { + this.ignoreWeightedRouting = isWeightedRoundRobinEnabled; + } + public boolean isIgnoreAwarenessAttr() { return ignoreAwarenessAttr; } @@ -314,11 +334,7 @@ private ShardIterator preferenceActiveShardIterator( } } preferenceType = Preference.parse(preference); - if (weightedRoutingMetadata != null && weightedRoutingMetadata.getWeightedRouting().isSet() && isStrictWeightedShardRouting) { - throw new PreferenceBasedSearchNotAllowedException( - "Preference type based routing not allowed with strict weighted shard routing enabled" - ); - } + checkPreferenceBasedRoutingAllowed(preferenceType, weightedRoutingMetadata); switch (preferenceType) { case PREFER_NODES: final Set nodesIds = Arrays.stream(preference.substring(Preference.PREFER_NODES.type().length() + 1).split(",")) @@ -344,11 +360,16 @@ private ShardIterator preferenceActiveShardIterator( // for a different element in the list by also incorporating the // shard ID into the hash of the user-supplied preference key. routingHash = 31 * routingHash + indexShard.shardId.hashCode(); - if (weightedRoutingMetadata != null && weightedRoutingMetadata.getWeightedRouting().isSet() && isStrictWeightedShardRouting) { - return indexShard.activeInitializingShardsSimpleWeightedIt( + if (WeightedRoutingUtils.shouldPerformStrictWeightedRouting( + isStrictWeightedShardRouting, + ignoreWeightedRouting, + weightedRoutingMetadata + )) { + return indexShard.activeInitializingShardsWeightedIt( weightedRoutingMetadata.getWeightedRouting(), nodes, getWeightedRoutingDefaultWeight(), + isFailOpenEnabled, routingHash ); } else if (ignoreAwarenessAttributes()) { @@ -365,12 +386,13 @@ private ShardIterator shardRoutings( @Nullable Map nodeCounts, @Nullable WeightedRoutingMetadata weightedRoutingMetadata ) { - if (weightedRoutingMetadata != null && weightedRoutingMetadata.getWeightedRouting().isSet()) { + if (WeightedRoutingUtils.shouldPerformWeightedRouting(ignoreWeightedRouting, weightedRoutingMetadata)) { return indexShard.activeInitializingShardsWeightedIt( weightedRoutingMetadata.getWeightedRouting(), nodes, getWeightedRoutingDefaultWeight(), - isFailOpenEnabled + isFailOpenEnabled, + null ); } else if (ignoreAwarenessAttributes()) { if (useAdaptiveReplicaSelection) { @@ -438,4 +460,15 @@ private static int calculateScaledShardId(IndexMetadata indexMetadata, String ef return Math.floorMod(hash, indexMetadata.getRoutingNumShards()) / indexMetadata.getRoutingFactor(); } + private void checkPreferenceBasedRoutingAllowed(Preference preference, @Nullable WeightedRoutingMetadata weightedRoutingMetadata) { + if (WeightedRoutingUtils.shouldPerformStrictWeightedRouting( + isStrictWeightedShardRouting, + ignoreWeightedRouting, + weightedRoutingMetadata + ) && WEIGHTED_ROUTING_RESTRICTED_PREFERENCES.contains(preference)) { + throw new PreferenceBasedSearchNotAllowedException( + "Preference type based routing not allowed with strict weighted shard routing enabled" + ); + } + } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java index 118d2998bae67..5cb0ecab1f27c 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java @@ -36,7 +36,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.CollectionUtil; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; diff --git a/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingUtils.java b/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingUtils.java index c7d40cbbbea61..72387aad0fa45 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingUtils.java +++ b/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingUtils.java @@ -53,4 +53,16 @@ public static boolean isWeighedAway(String nodeId, ClusterState clusterState) { } return false; } + + public static boolean shouldPerformWeightedRouting(boolean ignoreWeightedRouting, WeightedRoutingMetadata weightedRoutingMetadata) { + return !ignoreWeightedRouting && weightedRoutingMetadata != null && weightedRoutingMetadata.getWeightedRouting().isSet(); + } + + public static boolean shouldPerformStrictWeightedRouting( + boolean isStrictWeightedShardRouting, + boolean ignoreWeightedRouting, + WeightedRoutingMetadata weightedRoutingMetadata + ) { + return isStrictWeightedShardRouting && shouldPerformWeightedRouting(ignoreWeightedRouting, weightedRoutingMetadata); + } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 0ff0eeba7d394..6ba8e5d893bc0 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -162,7 +162,7 @@ private void setWeightFunction(float indexBalance, float shardBalanceFactor) { /** * When primary shards balance is desired, enable primary shard balancing constraints - * @param preferPrimaryShardBalance + * @param preferPrimaryShardBalance boolean to prefer balancing by primary shard */ private void setPreferPrimaryShardBalance(boolean preferPrimaryShardBalance) { this.preferPrimaryShardBalance = preferPrimaryShardBalance; diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java index ce6672f0961bd..a7d870513b107 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java @@ -57,7 +57,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.threadpool.Scheduler; diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java index 03bc3219ac8f8..9712fdbfbe8ec 100644 --- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.Version; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.AckedClusterStateTaskListener; @@ -63,7 +63,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.FutureUtils; import org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor; import org.opensearch.common.util.concurrent.ThreadContext; diff --git a/server/src/main/java/org/opensearch/cluster/service/TaskBatcher.java b/server/src/main/java/org/opensearch/cluster/service/TaskBatcher.java index ed744f4f42d07..ea8b74f15b8a9 100644 --- a/server/src/main/java/org/opensearch/cluster/service/TaskBatcher.java +++ b/server/src/main/java/org/opensearch/cluster/service/TaskBatcher.java @@ -36,7 +36,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Priority; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/common/blobstore/fs/FsBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/fs/FsBlobContainer.java index 940192d77630b..394855671688a 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/fs/FsBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/fs/FsBlobContainer.java @@ -40,7 +40,7 @@ import org.opensearch.common.blobstore.support.AbstractBlobContainer; import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.common.io.Streams; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.FileNotFoundException; import java.io.IOException; @@ -227,7 +227,7 @@ public void writeBlobAtomic(final String blobName, final InputStream inputStream private void writeToPath(InputStream inputStream, Path tempBlobPath, long blobSize) throws IOException { try (OutputStream outputStream = Files.newOutputStream(tempBlobPath, StandardOpenOption.CREATE_NEW)) { final int bufferSize = blobStore.bufferSizeInBytes(); - org.opensearch.core.internal.io.Streams.copy( + org.opensearch.common.util.io.Streams.copy( inputStream, outputStream, new byte[blobSize < bufferSize ? Math.toIntExact(blobSize) : bufferSize] diff --git a/server/src/main/java/org/opensearch/common/bytes/RecyclingBytesStreamOutput.java b/server/src/main/java/org/opensearch/common/bytes/RecyclingBytesStreamOutput.java index 28c3f4c9d5349..7f477a4ec6a53 100644 --- a/server/src/main/java/org/opensearch/common/bytes/RecyclingBytesStreamOutput.java +++ b/server/src/main/java/org/opensearch/common/bytes/RecyclingBytesStreamOutput.java @@ -39,7 +39,7 @@ import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.ByteArray; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/common/compress/DeflateCompressor.java b/server/src/main/java/org/opensearch/common/compress/DeflateCompressor.java index 05c5250ee32a4..36ee41e2d723d 100644 --- a/server/src/main/java/org/opensearch/common/compress/DeflateCompressor.java +++ b/server/src/main/java/org/opensearch/common/compress/DeflateCompressor.java @@ -32,7 +32,7 @@ package org.opensearch.common.compress; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/common/geo/builders/ShapeBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/ShapeBuilder.java index 5702eb9f5d85e..2ec70f5fc0098 100644 --- a/server/src/main/java/org/opensearch/common/geo/builders/ShapeBuilder.java +++ b/server/src/main/java/org/opensearch/common/geo/builders/ShapeBuilder.java @@ -38,7 +38,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.common.Strings; import org.opensearch.common.geo.GeoShapeType; import org.opensearch.common.geo.parsers.GeoWKTParser; diff --git a/server/src/main/java/org/opensearch/common/io/FileSystemUtils.java b/server/src/main/java/org/opensearch/common/io/FileSystemUtils.java index e65c430783013..383393d15b7de 100644 --- a/server/src/main/java/org/opensearch/common/io/FileSystemUtils.java +++ b/server/src/main/java/org/opensearch/common/io/FileSystemUtils.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.Constants; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.Strings; import org.opensearch.common.SuppressForbidden; diff --git a/server/src/main/java/org/opensearch/common/io/Streams.java b/server/src/main/java/org/opensearch/common/io/Streams.java index b4dab5ecb7e54..00e9c84e4c2aa 100644 --- a/server/src/main/java/org/opensearch/common/io/Streams.java +++ b/server/src/main/java/org/opensearch/common/io/Streams.java @@ -198,7 +198,7 @@ public static int readFully(InputStream reader, byte[] dest, int offset, int len * Fully consumes the input stream, throwing the bytes away. Returns the number of bytes consumed. */ public static long consumeFully(InputStream inputStream) throws IOException { - return org.opensearch.core.internal.io.Streams.copy(inputStream, NULL_OUTPUT_STREAM); + return org.opensearch.common.util.io.Streams.copy(inputStream, NULL_OUTPUT_STREAM); } public static List readAllLines(InputStream input) throws IOException { @@ -244,7 +244,7 @@ public static BytesStream flushOnCloseStream(BytesStream os) { */ public static BytesReference readFully(InputStream in) throws IOException { BytesStreamOutput out = new BytesStreamOutput(); - org.opensearch.core.internal.io.Streams.copy(in, out); + org.opensearch.common.util.io.Streams.copy(in, out); return out.bytes(); } diff --git a/server/src/main/java/org/opensearch/common/io/stream/StreamInput.java b/server/src/main/java/org/opensearch/common/io/stream/StreamInput.java index 4f3555fdb1852..419308880f04e 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/opensearch/common/io/stream/StreamInput.java @@ -55,7 +55,7 @@ import org.opensearch.common.text.Text; import org.opensearch.common.time.DateUtils; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.script.JodaCompatibleZonedDateTime; import java.io.ByteArrayInputStream; diff --git a/server/src/main/java/org/opensearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/opensearch/common/io/stream/StreamOutput.java index 11a22c4edf0c1..f3c184ea288bd 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/opensearch/common/io/stream/StreamOutput.java @@ -57,7 +57,7 @@ import org.opensearch.common.settings.SecureString; import org.opensearch.common.text.Text; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.script.JodaCompatibleZonedDateTime; import java.io.EOFException; diff --git a/server/src/main/java/org/opensearch/common/lease/Releasables.java b/server/src/main/java/org/opensearch/common/lease/Releasables.java index 5e885d239f884..31bfa9a2dd7ab 100644 --- a/server/src/main/java/org/opensearch/common/lease/Releasables.java +++ b/server/src/main/java/org/opensearch/common/lease/Releasables.java @@ -33,7 +33,7 @@ package org.opensearch.common.lease; import org.opensearch.common.Nullable; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java b/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java index 017aef1528243..f6f43cc214c5a 100644 --- a/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java +++ b/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java @@ -34,7 +34,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.index.shard.ShardId; import org.opensearch.index.shard.ShardUtils; diff --git a/server/src/main/java/org/opensearch/common/network/CloseableChannel.java b/server/src/main/java/org/opensearch/common/network/CloseableChannel.java index a849e0403a007..704b616bcabc9 100644 --- a/server/src/main/java/org/opensearch/common/network/CloseableChannel.java +++ b/server/src/main/java/org/opensearch/common/network/CloseableChannel.java @@ -35,7 +35,7 @@ import org.opensearch.action.ActionFuture; import org.opensearch.action.ActionListener; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.Closeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index edc5389798fac..b7979407f97a0 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -546,6 +546,7 @@ public void apply(Settings value, Settings current, Settings previous) { OperationRouting.WEIGHTED_ROUTING_DEFAULT_WEIGHT, OperationRouting.WEIGHTED_ROUTING_FAILOPEN_ENABLED, OperationRouting.STRICT_WEIGHTED_SHARD_ROUTING_ENABLED, + OperationRouting.IGNORE_WEIGHTED_SHARD_ROUTING, IndexGraveyard.SETTING_MAX_TOMBSTONES, PersistentTasksClusterService.CLUSTER_TASKS_ALLOCATION_RECHECK_INTERVAL_SETTING, EnableAssignmentDecider.CLUSTER_TASKS_ALLOCATION_ENABLE_SETTING, diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index fe4d4b02c05e7..ecb6f844831eb 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -56,9 +56,8 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.io.InputStream; diff --git a/server/src/main/java/org/opensearch/common/util/CollectionUtils.java b/server/src/main/java/org/opensearch/common/util/CollectionUtils.java index d8a86f4878f58..85d897af7479c 100644 --- a/server/src/main/java/org/opensearch/common/util/CollectionUtils.java +++ b/server/src/main/java/org/opensearch/common/util/CollectionUtils.java @@ -32,12 +32,6 @@ package org.opensearch.common.util; -import com.carrotsearch.hppc.ObjectArrayList; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefArray; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.InPlaceMergeSorter; -import org.apache.lucene.util.IntroSorter; import org.opensearch.common.Strings; import org.opensearch.common.collect.Iterators; @@ -50,7 +44,9 @@ import java.util.Comparator; import java.util.HashMap; import java.util.IdentityHashMap; +import java.util.Iterator; import java.util.List; +import java.util.ListIterator; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -95,59 +91,28 @@ public static List rotate(final List list, int distance) { return new RotatedList<>(list, d); } - public static void sortAndDedup(final ObjectArrayList array) { - int len = array.size(); - if (len > 1) { - sort(array); - int uniqueCount = 1; - for (int i = 1; i < len; ++i) { - if (!Arrays.equals(array.get(i), array.get(i - 1))) { - array.set(uniqueCount++, array.get(i)); - } - } - array.elementsCount = uniqueCount; + /** + * in place de-duplicates items in a list + */ + public static void sortAndDedup(final List array, Comparator comparator) { + // base case: one item + if (array.size() <= 1) { + return; } - } - - public static void sort(final ObjectArrayList array) { - new IntroSorter() { - - byte[] pivot; - - @Override - protected void swap(int i, int j) { - final byte[] tmp = array.get(i); - array.set(i, array.get(j)); - array.set(j, tmp); - } - - @Override - protected int compare(int i, int j) { - return compare(array.get(i), array.get(j)); - } - - @Override - protected void setPivot(int i) { - pivot = array.get(i); - } - - @Override - protected int comparePivot(int j) { - return compare(pivot, array.get(j)); - } - - private int compare(byte[] left, byte[] right) { - for (int i = 0, j = 0; i < left.length && j < right.length; i++, j++) { - int a = left[i] & 0xFF; - int b = right[j] & 0xFF; - if (a != b) { - return a - b; - } - } - return left.length - right.length; + array.sort(comparator); + ListIterator deduped = array.listIterator(); + T cmp = deduped.next(); // return the first item and advance + Iterator oldArray = array.iterator(); + oldArray.next(); // advance to the old to the second item (advanced to third below) + + do { + T old = oldArray.next(); // get the next item and advance iter + if (comparator.compare(cmp, old) != 0 && (cmp = deduped.next()) != old) { + deduped.set(old); } - - }.sort(0, array.size()); + } while (oldArray.hasNext()); + // in place update + array.subList(deduped.nextIndex(), array.size()).clear(); } public static int[] toArray(Collection ints) { @@ -252,65 +217,6 @@ public int size() { } } - public static void sort(final BytesRefArray bytes, final int[] indices) { - sort(new BytesRefBuilder(), new BytesRefBuilder(), bytes, indices); - } - - private static void sort( - final BytesRefBuilder scratch, - final BytesRefBuilder scratch1, - final BytesRefArray bytes, - final int[] indices - ) { - - final int numValues = bytes.size(); - assert indices.length >= numValues; - if (numValues > 1) { - new InPlaceMergeSorter() { - final Comparator comparator = Comparator.naturalOrder(); - - @Override - protected int compare(int i, int j) { - return comparator.compare(bytes.get(scratch, indices[i]), bytes.get(scratch1, indices[j])); - } - - @Override - protected void swap(int i, int j) { - int value_i = indices[i]; - indices[i] = indices[j]; - indices[j] = value_i; - } - }.sort(0, numValues); - } - - } - - public static int sortAndDedup(final BytesRefArray bytes, final int[] indices) { - final BytesRefBuilder scratch = new BytesRefBuilder(); - final BytesRefBuilder scratch1 = new BytesRefBuilder(); - final int numValues = bytes.size(); - assert indices.length >= numValues; - if (numValues <= 1) { - return numValues; - } - sort(scratch, scratch1, bytes, indices); - int uniqueCount = 1; - BytesRefBuilder previous = scratch; - BytesRefBuilder current = scratch1; - bytes.get(previous, indices[0]); - for (int i = 1; i < numValues; ++i) { - bytes.get(current, indices[i]); - if (!previous.get().equals(current.get())) { - indices[uniqueCount++] = indices[i]; - } - BytesRefBuilder tmp = previous; - previous = current; - current = tmp; - } - return uniqueCount; - - } - public static ArrayList iterableAsArrayList(Iterable elements) { if (elements == null) { throw new NullPointerException("elements"); diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchAbortPolicy.java b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchAbortPolicy.java index fc9a179083f59..2ea38e45d54d5 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchAbortPolicy.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchAbortPolicy.java @@ -33,6 +33,7 @@ package org.opensearch.common.util.concurrent; import org.opensearch.common.metrics.CounterMetric; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ThreadPoolExecutor; diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java index 7fda911fe7959..d967b7423ca80 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java @@ -33,6 +33,7 @@ package org.opensearch.common.util.concurrent; import org.opensearch.common.SuppressForbidden; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ThreadFactory; diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ReleasableLock.java b/server/src/main/java/org/opensearch/common/util/concurrent/ReleasableLock.java index e1aa6bd5d169a..883239cea9d99 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/ReleasableLock.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/ReleasableLock.java @@ -32,7 +32,7 @@ package org.opensearch.common.util.concurrent; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.common.lease.Releasable; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.engine.EngineException; diff --git a/server/src/main/java/org/opensearch/discovery/HandshakingTransportAddressConnector.java b/server/src/main/java/org/opensearch/discovery/HandshakingTransportAddressConnector.java index 8c0073cbbf245..90ca19e9369f4 100644 --- a/server/src/main/java/org/opensearch/discovery/HandshakingTransportAddressConnector.java +++ b/server/src/main/java/org/opensearch/discovery/HandshakingTransportAddressConnector.java @@ -46,7 +46,7 @@ import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.discovery.PeerFinder.TransportAddressConnector; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.ConnectionProfile; diff --git a/server/src/main/java/org/opensearch/env/NodeEnvironment.java b/server/src/main/java/org/opensearch/env/NodeEnvironment.java index ef3e8aa7e5583..1b43be128b052 100644 --- a/server/src/main/java/org/opensearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/opensearch/env/NodeEnvironment.java @@ -65,7 +65,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.gateway.MetadataStateFormat; import org.opensearch.gateway.PersistedClusterStateService; import org.opensearch.index.Index; diff --git a/server/src/main/java/org/opensearch/env/NodeRepurposeCommand.java b/server/src/main/java/org/opensearch/env/NodeRepurposeCommand.java index 35f2eead82159..f505a27a5624c 100644 --- a/server/src/main/java/org/opensearch/env/NodeRepurposeCommand.java +++ b/server/src/main/java/org/opensearch/env/NodeRepurposeCommand.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.gateway.MetadataStateFormat; import org.opensearch.gateway.PersistedClusterStateService; diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index ccc1bdb620f31..8488743051b6b 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import java.util.Arrays; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -586,34 +587,55 @@ private ExtensionsSettings readFromExtensionsYml(Path filePath) throws IOExcepti List> unreadExtensions = new ArrayList<>((Collection>) obj.get("extensions")); List readExtensions = new ArrayList(); for (HashMap extensionMap : unreadExtensions) { - // Parse extension dependencies - List extensionDependencyList = new ArrayList(); - if (extensionMap.get("dependencies") != null) { - List> extensionDependencies = new ArrayList<>( - (Collection>) extensionMap.get("dependencies") - ); - for (HashMap dependency : extensionDependencies) { - extensionDependencyList.add( - new ExtensionDependency( - dependency.get("uniqueId").toString(), - Version.fromString(dependency.get("version").toString()) - ) + try { + // checking to see whether any required fields are missing from extension.yml file or not + String[] requiredFields = { + "name", + "uniqueId", + "hostAddress", + "port", + "version", + "opensearchVersion", + "minimumCompatibleVersion" }; + List missingFields = Arrays.stream(requiredFields) + .filter(field -> !extensionMap.containsKey(field)) + .collect(Collectors.toList()); + if (!missingFields.isEmpty()) { + throw new IOException("Extension is missing these required fields : " + missingFields); + } + + // Parse extension dependencies + List extensionDependencyList = new ArrayList(); + if (extensionMap.get("dependencies") != null) { + List> extensionDependencies = new ArrayList<>( + (Collection>) extensionMap.get("dependencies") ); + for (HashMap dependency : extensionDependencies) { + extensionDependencyList.add( + new ExtensionDependency( + dependency.get("uniqueId").toString(), + Version.fromString(dependency.get("version").toString()) + ) + ); + } } + + // Create extension read from yml config + readExtensions.add( + new Extension( + extensionMap.get("name").toString(), + extensionMap.get("uniqueId").toString(), + extensionMap.get("hostAddress").toString(), + extensionMap.get("port").toString(), + extensionMap.get("version").toString(), + extensionMap.get("opensearchVersion").toString(), + extensionMap.get("minimumCompatibleVersion").toString(), + extensionDependencyList + ) + ); + } catch (IOException e) { + logger.warn("loading extension has been failed because of exception : " + e.getMessage()); } - // Create extension read from yml config - readExtensions.add( - new Extension( - extensionMap.get("name").toString(), - extensionMap.get("uniqueId").toString(), - extensionMap.get("hostAddress").toString(), - extensionMap.get("port").toString(), - extensionMap.get("version").toString(), - extensionMap.get("opensearchVersion").toString(), - extensionMap.get("minimumCompatibleVersion").toString(), - extensionDependencyList - ) - ); } inputStream.close(); return new ExtensionsSettings(readExtensions); diff --git a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestResponse.java b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestResponse.java index 9455df1bd0d6b..5cbc877f39eda 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestResponse.java +++ b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestResponse.java @@ -11,6 +11,7 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; import java.util.List; @@ -32,7 +33,7 @@ public class ExtensionRestResponse extends BytesRestResponse { * @param status The REST status. * @param builder The builder for the response. */ - public ExtensionRestResponse(ExtensionRestRequest request, RestStatus status, XContentBuilder builder) { + public ExtensionRestResponse(RestRequest request, RestStatus status, XContentBuilder builder) { super(status, builder); this.consumedParams = request.consumedParams(); this.contentConsumed = request.isContentConsumed(); @@ -45,7 +46,7 @@ public ExtensionRestResponse(ExtensionRestRequest request, RestStatus status, XC * @param status The REST status. * @param content A plain text response string. */ - public ExtensionRestResponse(ExtensionRestRequest request, RestStatus status, String content) { + public ExtensionRestResponse(RestRequest request, RestStatus status, String content) { super(status, content); this.consumedParams = request.consumedParams(); this.contentConsumed = request.isContentConsumed(); @@ -59,7 +60,7 @@ public ExtensionRestResponse(ExtensionRestRequest request, RestStatus status, St * @param contentType The content type of the response string. * @param content A response string. */ - public ExtensionRestResponse(ExtensionRestRequest request, RestStatus status, String contentType, String content) { + public ExtensionRestResponse(RestRequest request, RestStatus status, String contentType, String content) { super(status, contentType, content); this.consumedParams = request.consumedParams(); this.contentConsumed = request.isContentConsumed(); @@ -73,7 +74,7 @@ public ExtensionRestResponse(ExtensionRestRequest request, RestStatus status, St * @param contentType The content type of the response bytes. * @param content Response bytes. */ - public ExtensionRestResponse(ExtensionRestRequest request, RestStatus status, String contentType, byte[] content) { + public ExtensionRestResponse(RestRequest request, RestStatus status, String contentType, byte[] content) { super(status, contentType, content); this.consumedParams = request.consumedParams(); this.contentConsumed = request.isContentConsumed(); @@ -87,7 +88,7 @@ public ExtensionRestResponse(ExtensionRestRequest request, RestStatus status, St * @param contentType The content type of the response bytes. * @param content Response bytes. */ - public ExtensionRestResponse(ExtensionRestRequest request, RestStatus status, String contentType, BytesReference content) { + public ExtensionRestResponse(RestRequest request, RestStatus status, String contentType, BytesReference content) { super(status, contentType, content); this.consumedParams = request.consumedParams(); this.contentConsumed = request.isContentConsumed(); diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java index 6e359d690b1ec..b5c4d00228caa 100644 --- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java +++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java @@ -45,7 +45,7 @@ import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.common.Nullable; import org.opensearch.common.lease.Releasable; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.ShardId; import org.opensearch.transport.ReceiveTimeoutTransportException; diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index ea550fc6ac3e8..dcc3794320975 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -60,7 +60,7 @@ import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeMetadata; import org.opensearch.node.Node; import org.opensearch.plugins.MetadataUpgrader; diff --git a/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java index 6093be7ee663d..9943f04c964e1 100644 --- a/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java @@ -54,7 +54,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java index 74c0c2b998a19..d874b4a9375b8 100644 --- a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java @@ -86,7 +86,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.NodeMetadata; import org.opensearch.index.Index; diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 5e8f95acdfe39..a4ba510d013bd 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -55,7 +55,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.analysis.IndexAnalyzers; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 2ced9f56d7a35..eacb07f2e5c9c 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -40,7 +40,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Accountable; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; @@ -58,7 +58,7 @@ import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLock; import org.opensearch.env.ShardLockObtainFailedException; diff --git a/server/src/main/java/org/opensearch/index/IndexingPressure.java b/server/src/main/java/org/opensearch/index/IndexingPressure.java index a100f0b76f996..6285a06286e29 100644 --- a/server/src/main/java/org/opensearch/index/IndexingPressure.java +++ b/server/src/main/java/org/opensearch/index/IndexingPressure.java @@ -39,7 +39,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.stats.IndexingPressureStats; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java index 8e5fef2982607..563ada8d38924 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java @@ -18,8 +18,8 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.AbstractAsyncTask; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.AbstractAsyncTask; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ShardId; import org.opensearch.indices.IndicesService; diff --git a/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java b/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java index 59dde20c9a904..09e809c902ec6 100644 --- a/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java +++ b/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java @@ -12,7 +12,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.ShardIndexingPressureTracker.CommonOperationTracker; import org.opensearch.index.ShardIndexingPressureTracker.OperationTracker; import org.opensearch.index.ShardIndexingPressureTracker.PerformanceTracker; diff --git a/server/src/main/java/org/opensearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/opensearch/index/analysis/AnalysisRegistry.java index 4a7dc238a8c5d..cfdf416b2d533 100644 --- a/server/src/main/java/org/opensearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/opensearch/index/analysis/AnalysisRegistry.java @@ -39,7 +39,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.TextFieldMapper; diff --git a/server/src/main/java/org/opensearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/opensearch/index/analysis/IndexAnalyzers.java index 07be53d668b5e..4d72ee48cf33a 100644 --- a/server/src/main/java/org/opensearch/index/analysis/IndexAnalyzers.java +++ b/server/src/main/java/org/opensearch/index/analysis/IndexAnalyzers.java @@ -31,7 +31,7 @@ package org.opensearch.index.analysis; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.Closeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/analysis/PreBuiltAnalyzerProviderFactory.java b/server/src/main/java/org/opensearch/index/analysis/PreBuiltAnalyzerProviderFactory.java index 6202f0a679c83..1dafd4c5f6ca6 100644 --- a/server/src/main/java/org/opensearch/index/analysis/PreBuiltAnalyzerProviderFactory.java +++ b/server/src/main/java/org/opensearch/index/analysis/PreBuiltAnalyzerProviderFactory.java @@ -35,7 +35,7 @@ import org.apache.lucene.analysis.Analyzer; import org.opensearch.Version; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.indices.analysis.PreBuiltAnalyzers; diff --git a/server/src/main/java/org/opensearch/index/cache/IndexCache.java b/server/src/main/java/org/opensearch/index/cache/IndexCache.java index 0423a0bfd2706..90462e9919970 100644 --- a/server/src/main/java/org/opensearch/index/cache/IndexCache.java +++ b/server/src/main/java/org/opensearch/index/cache/IndexCache.java @@ -32,7 +32,7 @@ package org.opensearch.index.cache; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.AbstractIndexComponent; import org.opensearch.index.IndexSettings; import org.opensearch.index.cache.bitset.BitsetFilterCache; diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 55aad817a5816..a6fa86e379ebf 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -65,7 +65,7 @@ import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.InfoStream; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.ExceptionsHelper; import org.opensearch.action.index.IndexRequest; import org.opensearch.common.Booleans; @@ -86,7 +86,7 @@ import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.KeyedLock; import org.opensearch.common.util.concurrent.ReleasableLock; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.fieldvisitor.IdOnlyFieldVisitor; diff --git a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java index 098115bf676a8..d6995eab90fdb 100644 --- a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java +++ b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java @@ -50,7 +50,7 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.Queries; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.fieldvisitor.FieldsVisitor; import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.mapper.SourceFieldMapper; diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index 0ee58ab3cae28..a9bdcd1a64668 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -19,7 +19,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ReleasableLock; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.seqno.LocalCheckpointTracker; import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java index fbd3349240743..73ec658f573c2 100644 --- a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java @@ -43,7 +43,7 @@ import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.store.Store; diff --git a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java index 6965f3bf664ed..64f9a7e6f09cd 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java @@ -32,8 +32,9 @@ package org.opensearch.index.mapper; -import com.carrotsearch.hppc.ObjectArrayList; +import org.apache.lucene.document.InvertableType; import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StoredValue; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchException; @@ -52,6 +53,7 @@ import java.io.IOException; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Collections; @@ -240,39 +242,48 @@ protected String contentType() { */ public static class CustomBinaryDocValuesField extends CustomDocValuesField { - private final ObjectArrayList bytesList; - - private int totalSize = 0; + private final ArrayList bytesList; public CustomBinaryDocValuesField(String name, byte[] bytes) { super(name); - bytesList = new ObjectArrayList<>(); + bytesList = new ArrayList<>(); add(bytes); } public void add(byte[] bytes) { bytesList.add(bytes); - totalSize += bytes.length; } @Override public BytesRef binaryValue() { try { - CollectionUtils.sortAndDedup(bytesList); - int size = bytesList.size(); - BytesStreamOutput out = new BytesStreamOutput(totalSize + (size + 1) * 5); - out.writeVInt(size); // write total number of values - for (int i = 0; i < size; i++) { - final byte[] value = bytesList.get(i); - int valueLength = value.length; - out.writeVInt(valueLength); - out.writeBytes(value, 0, valueLength); + // sort and dedup in place + CollectionUtils.sortAndDedup(bytesList, Arrays::compareUnsigned); + int size = bytesList.stream().map(b -> b.length).reduce(0, Integer::sum); + int length = bytesList.size(); + try (BytesStreamOutput out = new BytesStreamOutput(size + (length + 1) * 5)) { + out.writeVInt(length); // write total number of values + for (byte[] value : bytesList) { + int valueLength = value.length; + out.writeVInt(valueLength); + out.writeBytes(value, 0, valueLength); + } + return out.bytes().toBytesRef(); } - return out.bytes().toBytesRef(); } catch (IOException e) { throw new OpenSearchException("Failed to get binary value", e); } } + + @Override + public StoredValue storedValue() { + return null; + } + + @Override + public InvertableType invertableType() { + return InvertableType.BINARY; + } } } diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index a5669bf9f99b4..521de037e63d0 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java index 5a8d8b648aa3c..01be8ee7a11c0 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java @@ -32,6 +32,8 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.InvertableType; +import org.apache.lucene.document.StoredValue; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -660,5 +662,15 @@ public BytesRef binaryValue() { throw new OpenSearchException("failed to encode ranges", e); } } + + @Override + public StoredValue storedValue() { + return null; + } + + @Override + public InvertableType invertableType() { + return InvertableType.BINARY; + } } } diff --git a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java index f01f90ef68bcd..286aa2cc93045 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java +++ b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java @@ -48,7 +48,7 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.mapper.RoutingFieldMapper; diff --git a/server/src/main/java/org/opensearch/index/reindex/WorkerBulkByScrollTaskState.java b/server/src/main/java/org/opensearch/index/reindex/WorkerBulkByScrollTaskState.java index eefdc4f90dc7b..e925e57784aa3 100644 --- a/server/src/main/java/org/opensearch/index/reindex/WorkerBulkByScrollTaskState.java +++ b/server/src/main/java/org/opensearch/index/reindex/WorkerBulkByScrollTaskState.java @@ -36,7 +36,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.RunOnce; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java index 9a121fe55a7e7..7d6cbc0064535 100644 --- a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java @@ -59,7 +59,7 @@ import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.Fuzziness; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DateFieldMapper.DateFieldType; import org.opensearch.index.mapper.FieldNamesFieldMapper; diff --git a/server/src/main/java/org/opensearch/index/shard/GlobalCheckpointListeners.java b/server/src/main/java/org/opensearch/index/shard/GlobalCheckpointListeners.java index 6dbbb62e231af..35dfcf1a0f46d 100644 --- a/server/src/main/java/org/opensearch/index/shard/GlobalCheckpointListeners.java +++ b/server/src/main/java/org/opensearch/index/shard/GlobalCheckpointListeners.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.common.collect.Tuple; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.FutureUtils; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 4a2d0f0198051..2629b2c954015 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -56,7 +56,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ThreadInterruptedException; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; @@ -99,7 +99,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.gateway.WriteStateException; import org.opensearch.index.Index; import org.opensearch.index.IndexModule; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShardOperationPermits.java b/server/src/main/java/org/opensearch/index/shard/IndexShardOperationPermits.java index 9d500db1d8036..b6af2030d7fb6 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShardOperationPermits.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShardOperationPermits.java @@ -32,7 +32,7 @@ package org.opensearch.index.shard; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionRunnable; @@ -43,7 +43,7 @@ import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.RunOnce; import org.opensearch.common.util.concurrent.ThreadContext.StoredContext; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.threadpool.ThreadPool; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java index a96e8015ccb27..5786c25031647 100644 --- a/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java +++ b/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java @@ -49,7 +49,7 @@ import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.translog.Translog; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/index/shard/ShardPath.java b/server/src/main/java/org/opensearch/index/shard/ShardPath.java index 66b7b4c7f46c5..39486ec00a36a 100644 --- a/server/src/main/java/org/opensearch/index/shard/ShardPath.java +++ b/server/src/main/java/org/opensearch/index/shard/ShardPath.java @@ -33,7 +33,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.util.Strings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.NodeEnvironment; diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/SlicedInputStream.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/SlicedInputStream.java index ae2e33d41c6ce..298b101a86299 100644 --- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/SlicedInputStream.java +++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/SlicedInputStream.java @@ -31,7 +31,7 @@ package org.opensearch.index.snapshots.blobstore; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.io.InputStream; diff --git a/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java index 88c7e12632863..3b5b4040954c9 100644 --- a/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java @@ -45,7 +45,7 @@ import org.apache.lucene.store.SimpleFSLockFactory; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 69de85cd23820..3a07ccf40831f 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -82,7 +82,7 @@ import org.opensearch.common.util.concurrent.AbstractRefCounted; import org.opensearch.common.util.concurrent.RefCounted; import org.opensearch.common.util.iterable.Iterables; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLock; import org.opensearch.env.ShardLockObtainFailedException; diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java index 976827d582745..2bb004028d015 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java +++ b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java @@ -22,6 +22,8 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; /** * This acts as entry point to fetch {@link BlobFetchRequest} and return actual {@link IndexInput}. Utilizes the BlobContainer interface to @@ -48,30 +50,36 @@ public TransferManager(final BlobContainer blobContainer, final FileCache fileCa public IndexInput fetchBlob(BlobFetchRequest blobFetchRequest) throws IOException { final Path key = blobFetchRequest.getFilePath(); - final IndexInput origin = fileCache.compute(key, (path, cachedIndexInput) -> { - if (cachedIndexInput == null) { - try { - return new FileCachedIndexInput(fileCache, blobFetchRequest.getFilePath(), downloadBlockLocally(blobFetchRequest)); - } catch (IOException e) { - logger.warn("Failed to download " + blobFetchRequest.getFilePath(), e); - return null; - } - } else { - if (cachedIndexInput.isClosed()) { - // if it's already in the file cache, but closed, open it and replace the original one + // We need to do a privileged action here in order to fetch from remote + // and write to the local file cache in case this is invoked as a side + // effect of a plugin (such as a scripted search) that doesn't have the + // necessary permissions. + final IndexInput origin = AccessController.doPrivileged( + (PrivilegedAction) () -> fileCache.compute(key, (path, cachedIndexInput) -> { + if (cachedIndexInput == null) { try { - final IndexInput luceneIndexInput = blobFetchRequest.getDirectory() - .openInput(blobFetchRequest.getFileName(), IOContext.READ); - return new FileCachedIndexInput(fileCache, blobFetchRequest.getFilePath(), luceneIndexInput); + return new FileCachedIndexInput(fileCache, blobFetchRequest.getFilePath(), downloadBlockLocally(blobFetchRequest)); } catch (IOException e) { - logger.warn("Failed to open existing file for " + blobFetchRequest.getFilePath(), e); + logger.warn("Failed to download " + blobFetchRequest.getFilePath(), e); return null; } + } else { + if (cachedIndexInput.isClosed()) { + // if it's already in the file cache, but closed, open it and replace the original one + try { + final IndexInput luceneIndexInput = blobFetchRequest.getDirectory() + .openInput(blobFetchRequest.getFileName(), IOContext.READ); + return new FileCachedIndexInput(fileCache, blobFetchRequest.getFilePath(), luceneIndexInput); + } catch (IOException e) { + logger.warn("Failed to open existing file for " + blobFetchRequest.getFilePath(), e); + return null; + } + } + // already in the cache and ready to be used (open) + return cachedIndexInput; } - // already in the cache and ready to be used (open) - return cachedIndexInput; - } - }); + }) + ); if (origin == null) { throw new IOException("Failed to create IndexInput for " + blobFetchRequest.getFileName()); diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java index 679b9d7f2f88b..0eb133eb464d2 100644 --- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java @@ -13,7 +13,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.store.AlreadyClosedException; import org.opensearch.common.util.concurrent.ReleasableLock; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.engine.LifecycleAware; import org.opensearch.index.seqno.LocalCheckpointTracker; import org.opensearch.index.shard.ShardId; diff --git a/server/src/main/java/org/opensearch/index/translog/LocalTranslog.java b/server/src/main/java/org/opensearch/index/translog/LocalTranslog.java index 404132f45f7cb..22dba3973cfc1 100644 --- a/server/src/main/java/org/opensearch/index/translog/LocalTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/LocalTranslog.java @@ -9,7 +9,7 @@ package org.opensearch.index.translog; import org.opensearch.common.util.concurrent.ReleasableLock; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.nio.file.Files; diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 5180469d83e1c..2230f13ad8c61 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -13,7 +13,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.concurrent.ReleasableLock; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.ShardId; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.index.translog.transfer.FileTransferTracker; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index f9ddb946b7e59..ba4d87ce84f07 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -48,7 +48,7 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.ReleasableLock; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.engine.Engine; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java b/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java index 95f39c1188b2b..fde6d6bbc0632 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java @@ -33,7 +33,7 @@ package org.opensearch.index.translog; import org.apache.lucene.util.Counter; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.common.lease.Releasable; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogReader.java b/server/src/main/java/org/opensearch/index/translog/TranslogReader.java index 205229949da77..c4a4fb7a460a0 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogReader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogReader.java @@ -34,7 +34,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.opensearch.common.io.Channels; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.seqno.SequenceNumbers; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index 1250184ff656b..3e2c7e448fdad 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -37,7 +37,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.bytes.BytesReference; @@ -50,7 +50,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.ReleasableLock; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.ShardId; diff --git a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java index ae45710a428fd..ef948cfade815 100644 --- a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java +++ b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java @@ -46,7 +46,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/FileSnapshot.java b/server/src/main/java/org/opensearch/index/translog/transfer/FileSnapshot.java index dabde2c032c69..239ef7c3c9300 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/FileSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/FileSnapshot.java @@ -11,7 +11,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.lucene.store.InputStreamIndexInput; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.BufferedInputStream; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 671eb5d72d574..9c83a33a37712 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -80,7 +80,7 @@ import org.opensearch.common.util.concurrent.AbstractRefCounted; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.iterable.Iterables; @@ -90,7 +90,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLock; import org.opensearch.env.ShardLockObtainFailedException; diff --git a/server/src/main/java/org/opensearch/indices/analysis/HunspellService.java b/server/src/main/java/org/opensearch/indices/analysis/HunspellService.java index 1832de5a6f6d7..ad56d6c9a6312 100644 --- a/server/src/main/java/org/opensearch/indices/analysis/HunspellService.java +++ b/server/src/main/java/org/opensearch/indices/analysis/HunspellService.java @@ -38,7 +38,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.NIOFSDirectory; import org.opensearch.OpenSearchException; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.io.FileSystemUtils; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/server/src/main/java/org/opensearch/indices/recovery/MultiChunkTransfer.java b/server/src/main/java/org/opensearch/indices/recovery/MultiChunkTransfer.java index 4cc75de7cd5d9..86e021bb6bfff 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/MultiChunkTransfer.java +++ b/server/src/main/java/org/opensearch/indices/recovery/MultiChunkTransfer.java @@ -34,12 +34,12 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.action.ActionListener; import org.opensearch.common.collect.Tuple; import org.opensearch.common.util.concurrent.AsyncIOProcessor; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.seqno.LocalCheckpointTracker; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java index 212b5c8bc3d96..052b86c901093 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java @@ -57,7 +57,7 @@ import org.opensearch.common.util.concurrent.FutureUtils; import org.opensearch.common.util.concurrent.ListenableFuture; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.engine.RecoveryEngineException; import org.opensearch.index.seqno.RetentionLease; import org.opensearch.index.seqno.RetentionLeaseNotFoundException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java index 9d7ca715979a4..7466ab5c1ca86 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.flush.FlushRequest; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java index 9637b4a791e5d..77ffd75d0148a 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java @@ -19,7 +19,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.concurrent.ConcurrentCollections; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.RemoteTransportException; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java index e95c2c6470b4b..5411b6b2c9068 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java @@ -22,7 +22,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.lucene.store.InputStreamIndexInput; import org.opensearch.common.util.CancellableThreads; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java index f9720991338c2..9aa16dc40ab06 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java @@ -19,7 +19,7 @@ import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.concurrent.ListenableFuture; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.recovery.DelayRecoveryException; diff --git a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java index 96e84fb69b573..5c34aba6632b2 100644 --- a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java +++ b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java @@ -56,7 +56,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java index 5863f3396b2ce..85a44aa905e08 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java @@ -43,7 +43,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 7fc4412e53d3e..91d43675bea75 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -52,7 +52,7 @@ import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.watcher.ResourceWatcherService; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.Build; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchTimeoutException; @@ -121,7 +121,7 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.discovery.Discovery; import org.opensearch.discovery.DiscoveryModule; import org.opensearch.env.Environment; diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java index b4446085243df..045a531deeb30 100644 --- a/server/src/main/java/org/opensearch/node/NodeService.java +++ b/server/src/main/java/org/opensearch/node/NodeService.java @@ -33,7 +33,7 @@ package org.opensearch.node; import org.opensearch.cluster.routing.WeightedRoutingStats; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.Build; import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.info.NodeInfo; diff --git a/server/src/main/java/org/opensearch/plugins/PluginSecurity.java b/server/src/main/java/org/opensearch/plugins/PluginSecurity.java index e783a53dcb82f..e7d92016d4082 100644 --- a/server/src/main/java/org/opensearch/plugins/PluginSecurity.java +++ b/server/src/main/java/org/opensearch/plugins/PluginSecurity.java @@ -36,7 +36,7 @@ import org.opensearch.cli.Terminal; import org.opensearch.cli.Terminal.Verbosity; import org.opensearch.cli.UserException; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.nio.file.Files; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 32bdb8b665520..c438abedfd37c 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -63,7 +63,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ConcurrentCollections; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java index fc23613c9913b..2554a1f376d4b 100644 --- a/server/src/main/java/org/opensearch/rest/RestController.java +++ b/server/src/main/java/org/opensearch/rest/RestController.java @@ -48,7 +48,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.Streams; +import org.opensearch.common.util.io.Streams; import org.opensearch.http.HttpServerTransport; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.usage.UsageService; diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java index eccaacc73c068..b4f70e359d618 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequest.java +++ b/server/src/main/java/org/opensearch/rest/RestRequest.java @@ -382,7 +382,7 @@ public Map params() { * * @return the list of currently consumed parameters. */ - List consumedParams() { + public List consumedParams() { return new ArrayList<>(consumedParams); } diff --git a/server/src/main/java/org/opensearch/script/ScriptService.java b/server/src/main/java/org/opensearch/script/ScriptService.java index 0eeb6b38e5b27..a7de3e2f354d3 100644 --- a/server/src/main/java/org/opensearch/script/ScriptService.java +++ b/server/src/main/java/org/opensearch/script/ScriptService.java @@ -53,7 +53,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.Closeable; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index 4bc40610facf2..b4278681346e3 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -49,7 +49,7 @@ import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.cache.bitset.BitsetFilterCache; diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index a15489cd4d127..cc1ee60dbb6bf 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -69,8 +69,8 @@ import org.opensearch.common.util.CollectionUtils; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.ConcurrentMapLong; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; diff --git a/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java b/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java index 2caacf33fcdcf..bded2417ba6c1 100644 --- a/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java +++ b/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java @@ -16,7 +16,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.SortedNumericSelector; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.comparators.NumericComparator; @@ -36,10 +35,9 @@ public class SortedWiderNumericSortField extends SortedNumericSortField { * @param field Name of field to sort by. Must not be null. * @param type Type of values * @param reverse True if natural order should be reversed. - * @param selector custom selector type for choosing the sort value from the set. */ - public SortedWiderNumericSortField(String field, Type type, boolean reverse, SortedNumericSelector.Type selector) { - super(field, type, reverse, selector); + public SortedWiderNumericSortField(String field, Type type, boolean reverse) { + super(field, type, reverse); } /** @@ -84,4 +82,23 @@ public int compareValues(Number first, Number second) { } }; } + + /** + * The only below types would be considered for widening during merging topDocs results for sort, + * This will support indices having different Numeric types to be sorted together. + * @param type SortField.Type + * @return returns true if type is supported for widened numeric comparisons + */ + public static boolean isTypeSupported(Type type) { + // Only below 4 numeric types supported as of now for widened merge + switch (type) { + case INT: + case LONG: + case FLOAT: + case DOUBLE: + return true; + default: + return false; + } + } } diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGenerator.java b/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGenerator.java index e9131015996d9..6335629f61cf1 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGenerator.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGenerator.java @@ -49,7 +49,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRefBuilder; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.CharArrayReader; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java index 86dbc31245b11..1aeda91629d4f 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java @@ -53,7 +53,7 @@ import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.engine.Engine; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.IndexEventListener; diff --git a/server/src/main/java/org/opensearch/tasks/TaskManager.java b/server/src/main/java/org/opensearch/tasks/TaskManager.java index ff760219716e6..519837311eb94 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskManager.java +++ b/server/src/main/java/org/opensearch/tasks/TaskManager.java @@ -37,7 +37,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchTimeoutException; diff --git a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java index 5374cd52e23b2..1feb115cb585a 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java @@ -53,12 +53,12 @@ import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.Streams; +import org.opensearch.common.util.io.Streams; import org.opensearch.threadpool.ThreadPool; import java.io.ByteArrayOutputStream; diff --git a/server/src/main/java/org/opensearch/threadpool/Scheduler.java b/server/src/main/java/org/opensearch/threadpool/Scheduler.java index 7c502216af31b..86c322ec89dd7 100644 --- a/server/src/main/java/org/opensearch/threadpool/Scheduler.java +++ b/server/src/main/java/org/opensearch/threadpool/Scheduler.java @@ -39,7 +39,7 @@ import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.OpenSearchAbortPolicy; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import java.util.concurrent.Delayed; import java.util.concurrent.Future; diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 1bf9cbd200b2b..104af6945dfe8 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -45,7 +45,7 @@ import org.opensearch.common.unit.SizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.concurrent.XRejectedExecutionHandler; @@ -410,7 +410,7 @@ public ExecutorService executor(String name) { * @return a ScheduledFuture who's get will return when the task is has been added to its target thread pool and throw an exception if * the task is canceled before it was added to its target thread pool. Once the task has been added to its target thread pool * the ScheduledFuture will cannot interact with it. - * @throws org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException if the task cannot be scheduled for execution + * @throws OpenSearchRejectedExecutionException if the task cannot be scheduled for execution */ @Override public ScheduledCancellable schedule(Runnable command, TimeValue delay, String executor) { diff --git a/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java b/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java index 5616980254adf..1a4854cfad8db 100644 --- a/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java +++ b/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java @@ -41,7 +41,7 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.concurrent.ListenableFuture; import org.opensearch.common.util.concurrent.RunOnce; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.util.Collections; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java b/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java index 5c11694431fc5..6877d9679eb81 100644 --- a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java +++ b/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java @@ -32,7 +32,7 @@ package org.opensearch.transport; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.io.Streams; diff --git a/server/src/main/java/org/opensearch/transport/InboundDecoder.java b/server/src/main/java/org/opensearch/transport/InboundDecoder.java index e2c35ee7aefb8..cbc583aec762c 100644 --- a/server/src/main/java/org/opensearch/transport/InboundDecoder.java +++ b/server/src/main/java/org/opensearch/transport/InboundDecoder.java @@ -38,7 +38,7 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; import java.util.function.Consumer; diff --git a/server/src/main/java/org/opensearch/transport/InboundMessage.java b/server/src/main/java/org/opensearch/transport/InboundMessage.java index 815f77779d139..1809b000f705c 100644 --- a/server/src/main/java/org/opensearch/transport/InboundMessage.java +++ b/server/src/main/java/org/opensearch/transport/InboundMessage.java @@ -36,7 +36,7 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java index e890929cfc29f..785faec543023 100644 --- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java @@ -49,7 +49,7 @@ import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java index 906513ca62249..7d6fe07666096 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java @@ -42,7 +42,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.threadpool.ThreadPool; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/transport/RemoteClusterService.java b/server/src/main/java/org/opensearch/transport/RemoteClusterService.java index d39f80ea6b383..838367baf8a2e 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/opensearch/transport/RemoteClusterService.java @@ -49,7 +49,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.CountDown; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.threadpool.ThreadPool; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java index 6a3063a835d5e..15aebd5f71123 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java @@ -45,7 +45,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.threadpool.ThreadPool; import java.io.Closeable; diff --git a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java index 8f2d2f37653d7..cbac3250d898e 100644 --- a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java +++ b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java @@ -54,7 +54,7 @@ import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/TransportLogger.java b/server/src/main/java/org/opensearch/transport/TransportLogger.java index 96eb3e8bf5fa4..c97b35a150e91 100644 --- a/server/src/main/java/org/opensearch/transport/TransportLogger.java +++ b/server/src/main/java/org/opensearch/transport/TransportLogger.java @@ -39,7 +39,7 @@ import org.opensearch.common.io.stream.InputStreamStreamInput; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index c73d38517276b..804dfab928dd4 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -56,9 +56,9 @@ import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.node.NodeClosedException; import org.opensearch.node.ReportingService; import org.opensearch.tasks.Task; diff --git a/server/src/test/java/org/opensearch/ExceptionsHelperTests.java b/server/src/test/java/org/opensearch/ExceptionsHelperTests.java index 41051d7ff2dd0..4c65eadf46e89 100644 --- a/server/src/test/java/org/opensearch/ExceptionsHelperTests.java +++ b/server/src/test/java/org/opensearch/ExceptionsHelperTests.java @@ -40,7 +40,7 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.ParsingException; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.Index; import org.opensearch.index.query.QueryShardException; import org.opensearch.index.shard.ShardId; diff --git a/server/src/test/java/org/opensearch/action/bulk/RetryTests.java b/server/src/test/java/org/opensearch/action/bulk/RetryTests.java index d3280ede6ce15..9da2d482ef5bd 100644 --- a/server/src/test/java/org/opensearch/action/bulk/RetryTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/RetryTests.java @@ -39,7 +39,7 @@ import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.update.UpdateRequest; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpClient; diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java index 6c23092c789d1..d53b860e6524a 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java @@ -56,7 +56,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.VersionType; diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java index 1d7828027baa0..d412b5383bc89 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java @@ -64,7 +64,7 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; diff --git a/server/src/test/java/org/opensearch/action/search/SearchPhaseExecutionExceptionTests.java b/server/src/test/java/org/opensearch/action/search/SearchPhaseExecutionExceptionTests.java index e1f94d79bfe4e..169f2d47743d1 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchPhaseExecutionExceptionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchPhaseExecutionExceptionTests.java @@ -38,7 +38,7 @@ import org.opensearch.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/test/java/org/opensearch/action/support/DefaultShardOperationFailedExceptionTests.java b/server/src/test/java/org/opensearch/action/support/DefaultShardOperationFailedExceptionTests.java index ceac5ed07f266..ee96ce5ad505c 100644 --- a/server/src/test/java/org/opensearch/action/support/DefaultShardOperationFailedExceptionTests.java +++ b/server/src/test/java/org/opensearch/action/support/DefaultShardOperationFailedExceptionTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/test/java/org/opensearch/action/support/RetryableActionTests.java b/server/src/test/java/org/opensearch/action/support/RetryableActionTests.java index fc6853ecdc9af..85eda47e9b797 100644 --- a/server/src/test/java/org/opensearch/action/support/RetryableActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/RetryableActionTests.java @@ -37,7 +37,7 @@ import org.opensearch.cluster.coordination.DeterministicTaskQueue; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java index 74810fa045790..f653f98840a56 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java @@ -56,7 +56,7 @@ import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.ShardId; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.NoneCircuitBreakerService; diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java index c5d4f3326746d..652f82409679b 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java @@ -54,7 +54,7 @@ import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ConcurrentCollections; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.set.Sets; import org.opensearch.index.shard.IndexShardNotStartedException; import org.opensearch.index.shard.IndexShardState; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index c4db0641717c6..23087e6dd2ba4 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -31,7 +31,7 @@ package org.opensearch.cluster.coordination; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.Version; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; diff --git a/server/src/test/java/org/opensearch/cluster/routing/FailAwareWeightedRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/FailAwareWeightedRoutingTests.java index 5c3a2454c4074..c325517837d09 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/FailAwareWeightedRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/FailAwareWeightedRoutingTests.java @@ -20,7 +20,7 @@ import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.ShardId; import org.opensearch.search.SearchShardTarget; import org.opensearch.test.OpenSearchTestCase; @@ -42,6 +42,10 @@ public class FailAwareWeightedRoutingTests extends OpenSearchTestCase { private ClusterState setUpCluster() { + return setUpCluster(Settings.EMPTY); + } + + private ClusterState setUpCluster(Settings transientSettings) { ClusterState clusterState = ClusterState.builder(new ClusterName("test")).build(); // set up nodes @@ -78,7 +82,7 @@ private ClusterState setUpCluster() { Map weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0); WeightedRouting weightedRouting = new WeightedRouting("zone", weights); WeightedRoutingMetadata weightedRoutingMetadata = new WeightedRoutingMetadata(weightedRouting, 0); - Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()); + Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()).transientSettings(transientSettings); metadataBuilder.putCustom(WeightedRoutingMetadata.TYPE, weightedRoutingMetadata); clusterState = ClusterState.builder(clusterState).metadata(metadataBuilder).build(); @@ -143,6 +147,124 @@ public void testFindNextWithoutFailOpen() throws IOException { assertEquals(1, shardSkipped.get()); } + public void testFindNextWithJustOneShardInStandbyZone() throws IOException { + ClusterState clusterState = setUpCluster(); + + AtomicInteger shardSkipped = new AtomicInteger(); + // set up index + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(SETTING_VERSION_CREATED, Version.CURRENT) + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 2) + .put(SETTING_CREATION_DATE, System.currentTimeMillis()) + ) + .build(); + + ShardRouting shardRoutingA = TestShardRouting.newShardRouting("test", 0, "node_zone_a", true, ShardRoutingState.STARTED); + ShardRouting shardRoutingB = TestShardRouting.newShardRouting("test", 0, "node_zone_b", false, ShardRoutingState.STARTED); + ShardRouting shardRoutingC = TestShardRouting.newShardRouting("test", 0, "node_zone_c", false, ShardRoutingState.STARTED); + + Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()); + metadataBuilder.put(indexMetadata, false).generateClusterUuidIfNeeded(); + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetadata.getIndex()); + + final ShardId shardId = new ShardId("test", "_na_", 0); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); + indexShardRoutingBuilder.addShard(shardRoutingA); + indexShardRoutingBuilder.addShard(shardRoutingB); + indexShardRoutingBuilder.addShard(shardRoutingC); + + indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build()); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + clusterState = ClusterState.builder(clusterState).routingTable(routingTableBuilder.build()).build(); + + List shardRoutings = new ArrayList<>(); + shardRoutings.add(shardRoutingC); + + String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); + SearchShardIterator searchShardIterator = new SearchShardIterator( + clusterAlias, + shardId, + shardRoutings, + OriginalIndicesTests.randomOriginalIndices() + ); + + // fail open is not executed since fail open conditions don't met + SearchShardTarget next = FailAwareWeightedRouting.getInstance() + .findNext(searchShardIterator, clusterState, new OpenSearchRejectedExecutionException(), () -> shardSkipped.incrementAndGet()); + assertNotNull(next); + next = FailAwareWeightedRouting.getInstance() + .findNext(searchShardIterator, clusterState, new OpenSearchRejectedExecutionException(), () -> shardSkipped.incrementAndGet()); + assertNull(next); + assertEquals(0, shardSkipped.get()); + } + + public void testFindNextWithIgnoreWeightedRoutingTrue() throws IOException { + ClusterState clusterState = setUpCluster(Settings.builder().put("cluster.routing.ignore_weighted_routing", true).build()); + + AtomicInteger shardSkipped = new AtomicInteger(); + // set up index + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(SETTING_VERSION_CREATED, Version.CURRENT) + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 2) + .put(SETTING_CREATION_DATE, System.currentTimeMillis()) + ) + .build(); + + ShardRouting shardRoutingA = TestShardRouting.newShardRouting("test", 0, "node_zone_a", true, ShardRoutingState.STARTED); + ShardRouting shardRoutingB = TestShardRouting.newShardRouting("test", 0, "node_zone_b", false, ShardRoutingState.STARTED); + ShardRouting shardRoutingC = TestShardRouting.newShardRouting("test", 0, "node_zone_c", false, ShardRoutingState.STARTED); + + Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()); + metadataBuilder.put(indexMetadata, false).generateClusterUuidIfNeeded(); + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetadata.getIndex()); + + final ShardId shardId = new ShardId("test", "_na_", 0); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); + indexShardRoutingBuilder.addShard(shardRoutingA); + indexShardRoutingBuilder.addShard(shardRoutingB); + indexShardRoutingBuilder.addShard(shardRoutingC); + + indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build()); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + clusterState = ClusterState.builder(clusterState).routingTable(routingTableBuilder.build()).build(); + + List shardRoutings = new ArrayList<>(); + shardRoutings.add(shardRoutingA); + shardRoutings.add(shardRoutingB); + shardRoutings.add(shardRoutingC); + + String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); + SearchShardIterator searchShardIterator = new SearchShardIterator( + clusterAlias, + shardId, + shardRoutings, + OriginalIndicesTests.randomOriginalIndices() + ); + + // fail open is not executed since fail open conditions don't met + SearchShardTarget next = FailAwareWeightedRouting.getInstance() + .findNext(searchShardIterator, clusterState, new OpenSearchRejectedExecutionException(), () -> shardSkipped.incrementAndGet()); + assertNotNull(next); + next = FailAwareWeightedRouting.getInstance() + .findNext(searchShardIterator, clusterState, new OpenSearchRejectedExecutionException(), () -> shardSkipped.incrementAndGet()); + assertNotNull(next); + next = FailAwareWeightedRouting.getInstance() + .findNext(searchShardIterator, clusterState, new OpenSearchRejectedExecutionException(), () -> shardSkipped.incrementAndGet()); + assertNotNull(next); + next = FailAwareWeightedRouting.getInstance() + .findNext(searchShardIterator, clusterState, new OpenSearchRejectedExecutionException(), () -> shardSkipped.incrementAndGet()); + assertNull(next); + assertEquals(0, shardSkipped.get()); + } + public void testFindNextWithFailOpenDueTo5xx() throws IOException { ClusterState clusterState = setUpCluster(); diff --git a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java index ffdb2d39fb817..e9cbb42369a19 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java @@ -44,7 +44,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.Index; import org.opensearch.index.shard.ShardId; import org.opensearch.node.ResponseCollectorService; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java index f5a418bc6a100..e1f5eba735904 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java @@ -365,7 +365,6 @@ public void testPrimaryBalanceWithContrainstBreaching() { /** * This test verifies global balance by creating indices iteratively and verify primary shards do not pile up on one * node. - * @throws Exception */ public void testGlobalPrimaryBalance() throws Exception { AllocationService strategy = createAllocationService(getSettingsBuilderForPrimaryBalance().build(), new TestGatewayAllocator()); diff --git a/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java b/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java index 866939e6ac3d7..59ea0dfca559a 100644 --- a/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java +++ b/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java @@ -552,7 +552,7 @@ public void testWeightedRoutingWithDifferentWeights() { ShardIterator shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null); assertEquals(2, shardIterator.size()); ShardRouting shardRouting; @@ -565,7 +565,7 @@ public void testWeightedRoutingWithDifferentWeights() { shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null); assertEquals(3, shardIterator.size()); weights = Map.of("zone1", -1.0, "zone2", 0.0, "zone3", 1.0); @@ -573,7 +573,7 @@ public void testWeightedRoutingWithDifferentWeights() { shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null); assertEquals(1, shardIterator.size()); shardRouting = shardIterator.nextOrNull(); assertNotNull(shardRouting); @@ -584,7 +584,7 @@ public void testWeightedRoutingWithDifferentWeights() { shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null); assertEquals(3, shardIterator.size()); shardRouting = shardIterator.nextOrNull(); assertNotNull(shardRouting); @@ -646,7 +646,7 @@ public void testWeightedRoutingInMemoryStore() { ShardIterator shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null); assertEquals(2, shardIterator.size()); ShardRouting shardRouting; shardRouting = shardIterator.nextOrNull(); @@ -660,7 +660,7 @@ public void testWeightedRoutingInMemoryStore() { shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null); assertEquals(2, shardIterator.size()); shardRouting = shardIterator.nextOrNull(); assertNotNull(shardRouting); @@ -675,7 +675,7 @@ public void testWeightedRoutingInMemoryStore() { shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null); assertEquals(2, shardIterator.size()); shardRouting = shardIterator.nextOrNull(); assertNotNull(shardRouting); @@ -690,7 +690,7 @@ public void testWeightedRoutingInMemoryStore() { shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null); assertEquals(2, shardIterator.size()); shardRouting = shardIterator.nextOrNull(); assertNotNull(shardRouting); @@ -755,7 +755,7 @@ public void testWeightedRoutingShardState() { ShardIterator shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null); assertEquals(3, shardIterator.size()); ShardRouting shardRouting; @@ -834,21 +834,21 @@ public void testWeightedRoutingShardStateWithDifferentWeights() { ShardIterator shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null); ShardRouting shardRouting1 = shardIterator.nextOrNull(); shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null); ShardRouting shardRouting2 = shardIterator.nextOrNull(); shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null); ShardRouting shardRouting3 = shardIterator.nextOrNull(); @@ -864,8 +864,8 @@ public void testWeightedRoutingShardStateWithDifferentWeights() { /** * Test to validate that simple weighted shard routing with seed return same shard routing on each call */ - public void testActiveInitializingShardsSimpleWeightedIt() { - TestThreadPool threadPool = new TestThreadPool("testActiveInitializingShardsSimpleWeightedIt"); + public void testActiveInitializingShardsWeightedItWithCustomSeed() { + TestThreadPool threadPool = new TestThreadPool("testActiveInitializingShardsWeightedItWithCustomSeed"); try { Settings.Builder settings = Settings.builder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) @@ -912,16 +912,15 @@ public void testActiveInitializingShardsSimpleWeightedIt() { ShardIterator shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsSimpleWeightedIt(weightedRouting, clusterState.nodes(), 1, 1); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, 1); ShardRouting shardRouting1 = shardIterator.nextOrNull(); for (int i = 0; i < 50; i++) { - shardIterator = clusterState.routingTable() .index("test") .shard(0) - .activeInitializingShardsSimpleWeightedIt(weightedRouting, clusterState.nodes(), 1, 1); + .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, 1); ShardRouting shardRouting2 = shardIterator.nextOrNull(); diff --git a/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java b/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java index 83f5c0445e21d..6c36368bfe446 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/fs/FsBlobContainerTests.java @@ -38,7 +38,7 @@ import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.PathUtilsForTesting; import org.opensearch.common.io.Streams; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/common/hppc/HppcMapsTests.java b/server/src/test/java/org/opensearch/common/hppc/HppcMapsTests.java index de3c2cf909383..42450bb69ce75 100644 --- a/server/src/test/java/org/opensearch/common/hppc/HppcMapsTests.java +++ b/server/src/test/java/org/opensearch/common/hppc/HppcMapsTests.java @@ -32,7 +32,7 @@ package org.opensearch.common.hppc; import com.carrotsearch.hppc.ObjectHashSet; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.common.collect.HppcMaps; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/io/StreamsTests.java b/server/src/test/java/org/opensearch/common/io/StreamsTests.java index f28d1b1762ca2..c52ce9703cc21 100644 --- a/server/src/test/java/org/opensearch/common/io/StreamsTests.java +++ b/server/src/test/java/org/opensearch/common/io/StreamsTests.java @@ -104,7 +104,7 @@ public void testLimitInputStream() throws IOException { final int limit = randomIntBetween(0, bytes.length); final BytesArray stuffArray = new BytesArray(bytes); final ByteArrayOutputStream out = new ByteArrayOutputStream(bytes.length); - final long count = org.opensearch.core.internal.io.Streams.copy(Streams.limitStream(stuffArray.streamInput(), limit), out); + final long count = org.opensearch.common.util.io.Streams.copy(Streams.limitStream(stuffArray.streamInput(), limit), out); assertEquals(limit, count); assertThat(Arrays.equals(out.toByteArray(), Arrays.copyOf(bytes, limit)), equalTo(true)); } diff --git a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java index e7756cbd96734..f61338af2ba8a 100644 --- a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java @@ -82,7 +82,7 @@ import org.opensearch.Version; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.opensearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; diff --git a/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java b/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java index 025da248565fc..60f39208d008a 100644 --- a/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java @@ -48,7 +48,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.util.BigArrays; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReaderTests.java b/server/src/test/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReaderTests.java index 1a2e23d43b579..cf4cc5cd56f68 100644 --- a/server/src/test/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReaderTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReaderTests.java @@ -41,7 +41,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/util/CollectionUtilsTests.java b/server/src/test/java/org/opensearch/common/util/CollectionUtilsTests.java index 40b2706d314ce..f0a0d41aa7aab 100644 --- a/server/src/test/java/org/opensearch/common/util/CollectionUtilsTests.java +++ b/server/src/test/java/org/opensearch/common/util/CollectionUtilsTests.java @@ -32,28 +32,21 @@ package org.opensearch.common.util; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefArray; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.Counter; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; import static java.util.Collections.emptyMap; import static org.opensearch.common.util.CollectionUtils.eagerPartition; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; public class CollectionUtilsTests extends OpenSearchTestCase { public void testRotateEmpty() { @@ -85,65 +78,30 @@ public void testRotate() { } } - public void testSortAndDedupByteRefArray() { - SortedSet set = new TreeSet<>(); - final int numValues = scaledRandomIntBetween(0, 10000); - List tmpList = new ArrayList<>(); - BytesRefArray array = new BytesRefArray(Counter.newCounter()); - for (int i = 0; i < numValues; i++) { - String s = randomRealisticUnicodeOfCodepointLengthBetween(1, 100); - set.add(new BytesRef(s)); - tmpList.add(new BytesRef(s)); - array.append(new BytesRef(s)); - } - if (randomBoolean()) { - Collections.shuffle(tmpList, random()); - for (BytesRef ref : tmpList) { - array.append(ref); + private void assertDeduped(List array, Comparator cmp, int expectedLength) { + // test the dedup w/ ArrayLists and LinkedLists + List> types = List.of(new ArrayList(array), new LinkedList<>(array)); + for (List clone : types) { + // dedup the list + CollectionUtils.sortAndDedup(clone, cmp); + // verify unique elements + for (int i = 0; i < clone.size() - 1; ++i) { + assertNotEquals(cmp.compare(clone.get(i), clone.get(i + 1)), 0); } + assertEquals(expectedLength, clone.size()); } - int[] indices = new int[array.size()]; - for (int i = 0; i < indices.length; i++) { - indices[i] = i; - } - int numUnique = CollectionUtils.sortAndDedup(array, indices); - assertThat(numUnique, equalTo(set.size())); - Iterator iterator = set.iterator(); - - BytesRefBuilder spare = new BytesRefBuilder(); - for (int i = 0; i < numUnique; i++) { - assertThat(iterator.hasNext(), is(true)); - assertThat(array.get(spare, indices[i]), equalTo(iterator.next())); - } - } - public void testSortByteRefArray() { - List values = new ArrayList<>(); - final int numValues = scaledRandomIntBetween(0, 10000); - BytesRefArray array = new BytesRefArray(Counter.newCounter()); - for (int i = 0; i < numValues; i++) { - String s = randomRealisticUnicodeOfCodepointLengthBetween(1, 100); - values.add(new BytesRef(s)); - array.append(new BytesRef(s)); - } - if (randomBoolean()) { - Collections.shuffle(values, random()); - } - int[] indices = new int[array.size()]; - for (int i = 0; i < indices.length; i++) { - indices[i] = i; - } - CollectionUtils.sort(array, indices); - Collections.sort(values); - Iterator iterator = values.iterator(); - - BytesRefBuilder spare = new BytesRefBuilder(); - for (int i = 0; i < values.size(); i++) { - assertThat(iterator.hasNext(), is(true)); - assertThat(array.get(spare, indices[i]), equalTo(iterator.next())); - } - + public void testSortAndDedup() { + // test no elements in a string array + assertDeduped(List.of(), Comparator.naturalOrder(), 0); + // test no elements in an integer array + assertDeduped(List.of(), Comparator.naturalOrder(), 0); + // test unsorted array + assertDeduped(List.of(-1, 0, 2, 1, -1, 19, -1), Comparator.naturalOrder(), 5); + // test sorted array + assertDeduped(List.of(-1, 0, 1, 2, 19, 19), Comparator.naturalOrder(), 5); + // test sorted } public void testEmptyPartition() { diff --git a/server/src/test/java/org/opensearch/common/util/concurrent/AbstractAsyncTaskTests.java b/server/src/test/java/org/opensearch/common/util/concurrent/AbstractAsyncTaskTests.java index b62b830de5036..e56a1793c06e6 100644 --- a/server/src/test/java/org/opensearch/common/util/concurrent/AbstractAsyncTaskTests.java +++ b/server/src/test/java/org/opensearch/common/util/concurrent/AbstractAsyncTaskTests.java @@ -33,7 +33,7 @@ import org.opensearch.common.Randomness; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/common/util/concurrent/OpenSearchExecutorsTests.java b/server/src/test/java/org/opensearch/common/util/concurrent/OpenSearchExecutorsTests.java index 61f489f32adfd..2063cd26a9e8e 100644 --- a/server/src/test/java/org/opensearch/common/util/concurrent/OpenSearchExecutorsTests.java +++ b/server/src/test/java/org/opensearch/common/util/concurrent/OpenSearchExecutorsTests.java @@ -34,6 +34,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matcher; diff --git a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java index e91da4f5ee46e..0f831ff28af79 100644 --- a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java +++ b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java @@ -111,7 +111,10 @@ public void testVectorHighlighterPrefixQuery() throws Exception { FastVectorHighlighter highlighter = new FastVectorHighlighter(); PrefixQuery prefixQuery = new PrefixQuery(new Term("content", "ba")); - assertThat(prefixQuery.getRewriteMethod().getClass().getName(), equalTo(PrefixQuery.CONSTANT_SCORE_REWRITE.getClass().getName())); + assertThat( + prefixQuery.getRewriteMethod().getClass().getName(), + equalTo(PrefixQuery.CONSTANT_SCORE_BLENDED_REWRITE.getClass().getName()) + ); String fragment = highlighter.getBestFragment( highlighter.getFieldQuery(prefixQuery), reader, @@ -128,7 +131,10 @@ public void testVectorHighlighterPrefixQuery() throws Exception { // now check with the custom field query prefixQuery = new PrefixQuery(new Term("content", "ba")); - assertThat(prefixQuery.getRewriteMethod().getClass().getName(), equalTo(PrefixQuery.CONSTANT_SCORE_REWRITE.getClass().getName())); + assertThat( + prefixQuery.getRewriteMethod().getClass().getName(), + equalTo(PrefixQuery.CONSTANT_SCORE_BLENDED_REWRITE.getClass().getName()) + ); fragment = highlighter.getBestFragment( new CustomFieldQuery(prefixQuery, reader, highlighter), reader, diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index 1be312194d336..2a2e95e529c7e 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.gateway.GatewayMetaState; import org.opensearch.plugins.DiscoveryPlugin; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java index 53e7f59460e47..60800ee6b64eb 100644 --- a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java +++ b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java @@ -46,7 +46,7 @@ import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.concurrent.FutureUtils; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java index 8e997ae6d37ae..4c07fc2619e7f 100644 --- a/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.set.Sets; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.gateway.MetadataStateFormat; import org.opensearch.index.Index; import org.opensearch.index.IndexSettings; diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index bbc115307e26f..040c799efce2a 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -281,6 +281,58 @@ public void testNonUniqueExtensionsDiscovery() throws Exception { assertTrue(expectedExtensions.containsAll(emptyList())); } + public void testMissingRequiredFieldsInExtensionDiscovery() throws Exception { + Path emptyExtensionDir = createTempDir(); + ExtensionsManager extensionsManager; + List requiredFieldMissingYmlLines = extensionsYmlLines.stream() + .map(s -> s.replace(" minimumCompatibleVersion: '2.0.0'", "")) + .collect(Collectors.toList()); + Files.write(emptyExtensionDir.resolve("extensions.yml"), requiredFieldMissingYmlLines, StandardCharsets.UTF_8); + + try (MockLogAppender mockLogAppender = MockLogAppender.createForLoggers(LogManager.getLogger(ExtensionsManager.class))) { + + mockLogAppender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "Required field is missing in extensions.yml", + "org.opensearch.extensions.ExtensionsManager", + Level.WARN, + "loading extension has been failed because of exception : Extension is missing these required fields : [minimumCompatibleVersion]" + ) + ); + + extensionsManager = new ExtensionsManager(settings, emptyExtensionDir); + + mockLogAppender.assertAllExpectationsMatched(); + } + + List expectedExtensions = new ArrayList(); + + expectedExtensions.add( + new DiscoveryExtensionNode( + "firstExtension", + "uniqueid1", + new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), + new HashMap(), + Version.fromString("3.0.0"), + Version.fromString("3.0.0"), + Collections.emptyList() + ) + ); + assertEquals(expectedExtensions.size(), extensionsManager.getExtensionIdMap().values().size()); + for (DiscoveryExtensionNode extension : expectedExtensions) { + DiscoveryExtensionNode initializedExtension = extensionsManager.getExtensionIdMap().get(extension.getId()); + assertEquals(extension.getName(), initializedExtension.getName()); + assertEquals(extension.getId(), initializedExtension.getId()); + assertEquals(extension.getAddress(), initializedExtension.getAddress()); + assertEquals(extension.getAttributes(), initializedExtension.getAttributes()); + assertEquals(extension.getVersion(), initializedExtension.getVersion()); + assertEquals(extension.getMinimumCompatibleVersion(), initializedExtension.getMinimumCompatibleVersion()); + assertEquals(extension.getDependencies(), initializedExtension.getDependencies()); + } + assertTrue(expectedExtensions.containsAll(emptyList())); + assertTrue(expectedExtensions.containsAll(emptyList())); + } + public void testDiscoveryExtension() throws Exception { String expectedId = "test id"; Version expectedVersion = Version.fromString("2.0.0"); diff --git a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestResponseTests.java b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestResponseTests.java index 5d5d3769a9427..cdc79e9778e2d 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestResponseTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestResponseTests.java @@ -11,11 +11,18 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; +import java.util.List; +import java.util.Map; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.http.HttpRequest; +import org.opensearch.http.HttpResponse; +import org.opensearch.http.HttpRequest.HttpVersion; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestStatus; import org.opensearch.rest.RestRequest.Method; import org.opensearch.test.OpenSearchTestCase; @@ -38,15 +45,63 @@ public void setUp() throws Exception { testBytes = new byte[] { 1, 2 }; } - private ExtensionRestRequest generateTestRequest() { - ExtensionRestRequest request = new ExtensionRestRequest( - Method.GET, - "/foo", - Collections.emptyMap(), - null, - new BytesArray("Text Content"), - null - ); + private RestRequest generateTestRequest() { + RestRequest request = RestRequest.request(null, new HttpRequest() { + + @Override + public Method method() { + return Method.GET; + } + + @Override + public String uri() { + return "/foo"; + } + + @Override + public BytesReference content() { + return new BytesArray("Text Content"); + } + + @Override + public Map> getHeaders() { + return Collections.emptyMap(); + } + + @Override + public List strictCookies() { + return Collections.emptyList(); + } + + @Override + public HttpVersion protocolVersion() { + return null; + } + + @Override + public HttpRequest removeHeader(String header) { + // we don't use + return null; + } + + @Override + public HttpResponse createResponse(RestStatus status, BytesReference content) { + return null; + } + + @Override + public Exception getInboundException() { + return null; + } + + @Override + public void release() {} + + @Override + public HttpRequest releaseAndCopy() { + return null; + } + }, null); // consume params "foo" and "bar" request.param("foo"); request.param("bar"); @@ -60,7 +115,7 @@ public void testConstructorWithBuilder() throws IOException { builder.startObject(); builder.field("status", ACCEPTED); builder.endObject(); - ExtensionRestRequest request = generateTestRequest(); + RestRequest request = generateTestRequest(); ExtensionRestResponse response = new ExtensionRestResponse(request, OK, builder); assertEquals(OK, response.status()); @@ -73,7 +128,7 @@ public void testConstructorWithBuilder() throws IOException { } public void testConstructorWithPlainText() { - ExtensionRestRequest request = generateTestRequest(); + RestRequest request = generateTestRequest(); ExtensionRestResponse response = new ExtensionRestResponse(request, OK, testText); assertEquals(OK, response.status()); @@ -86,7 +141,7 @@ public void testConstructorWithPlainText() { } public void testConstructorWithText() { - ExtensionRestRequest request = generateTestRequest(); + RestRequest request = generateTestRequest(); ExtensionRestResponse response = new ExtensionRestResponse(request, OK, TEXT_CONTENT_TYPE, testText); assertEquals(OK, response.status()); @@ -100,7 +155,7 @@ public void testConstructorWithText() { } public void testConstructorWithByteArray() { - ExtensionRestRequest request = generateTestRequest(); + RestRequest request = generateTestRequest(); ExtensionRestResponse response = new ExtensionRestResponse(request, OK, OCTET_CONTENT_TYPE, testBytes); assertEquals(OK, response.status()); @@ -113,7 +168,7 @@ public void testConstructorWithByteArray() { } public void testConstructorWithBytesReference() { - ExtensionRestRequest request = generateTestRequest(); + RestRequest request = generateTestRequest(); ExtensionRestResponse response = new ExtensionRestResponse( request, OK, diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index cc9a7a70b99f4..830a8a9ad8ab7 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -54,7 +54,7 @@ import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.util.set.Sets; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.TestEnvironment; diff --git a/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java index d7b8712a46f66..2b8cec86db5a2 100644 --- a/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java @@ -56,7 +56,7 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.NodeMetadata; diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index ad6344882b70d..d9d87196ca289 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -63,9 +63,9 @@ import org.opensearch.common.settings.SettingsException; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLock; diff --git a/server/src/test/java/org/opensearch/index/IndexingPressureTests.java b/server/src/test/java/org/opensearch/index/IndexingPressureTests.java index 0f9d9c7be1395..1aa20506222d1 100644 --- a/server/src/test/java/org/opensearch/index/IndexingPressureTests.java +++ b/server/src/test/java/org/opensearch/index/IndexingPressureTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.stats.IndexingPressureStats; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java index 3bc84c2c44be8..1ebdd111bfed3 100644 --- a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java +++ b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java @@ -16,7 +16,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; import org.opensearch.index.shard.IndexShard; diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java index 8757458e3317e..170919fbf351c 100644 --- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java +++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java @@ -16,7 +16,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.ShardId; import org.opensearch.index.stats.IndexingPressurePerShardStats; import org.opensearch.index.stats.IndexingPressureStats; diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java index 5722ebdf27fb5..77593660a7d12 100644 --- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java +++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java @@ -12,7 +12,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.ShardId; import org.opensearch.index.stats.IndexingPressurePerShardStats; import org.opensearch.index.stats.IndexingPressureStats; diff --git a/server/src/test/java/org/opensearch/index/cache/bitset/BitSetFilterCacheTests.java b/server/src/test/java/org/opensearch/index/cache/bitset/BitSetFilterCacheTests.java index 1f5833ef37fb3..38b5bf8d81f66 100644 --- a/server/src/test/java/org/opensearch/index/cache/bitset/BitSetFilterCacheTests.java +++ b/server/src/test/java/org/opensearch/index/cache/bitset/BitSetFilterCacheTests.java @@ -52,7 +52,7 @@ import org.apache.lucene.util.BitSet; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java index 74ce2d5bba186..b746d0ba8a56d 100644 --- a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java @@ -43,7 +43,7 @@ import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; import org.opensearch.OpenSearchException; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.search.suggest.completion.CompletionStats; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index cc1aaa02853e2..4b0f55f47d4cf 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -41,6 +41,7 @@ import org.apache.logging.log4j.core.appender.AbstractAppender; import org.apache.logging.log4j.core.filter.RegexFilter; import org.apache.lucene.document.Field; +import org.apache.lucene.document.KeywordField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; @@ -51,6 +52,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.LiveIndexWriterConfig; @@ -118,10 +120,11 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.codec.CodecService; +import org.opensearch.index.engine.Engine.IndexResult; import org.opensearch.index.fieldvisitor.FieldsVisitor; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.ParseContext; @@ -3935,13 +3938,13 @@ public void testHandleDocumentFailure() throws Exception { assertNotNull(indexResult.getTranslogLocation()); engine.index(indexForDoc(doc2)); - // test non document level failure is thrown - if (randomBoolean()) { - // simulate close by corruption - throwingIndexWriter.get().setThrowFailure(null); - UncheckedIOException uncheckedIOException = expectThrows(UncheckedIOException.class, () -> { + try (engine) { + // test non document level failure is thrown + if (randomBoolean()) { + // simulate close by corruption + throwingIndexWriter.get().setThrowFailure(null); Engine.Index index = indexForDoc(doc3); - index.parsedDoc().rootDoc().add(new StoredField("foo", "bar") { + index.parsedDoc().rootDoc().add(new KeywordField("foo", "bar", org.apache.lucene.document.Field.Store.YES) { // this is a hack to add a failure during store document which triggers a tragic event // and in turn fails the engine @Override @@ -3949,13 +3952,14 @@ public BytesRef binaryValue() { throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException()); } }); - engine.index(index); - }); - assertTrue(uncheckedIOException.getCause() instanceof MockDirectoryWrapper.FakeIOException); - } else { - // normal close - engine.close(); + + final IndexResult r = engine.index(index); + assertThat(r.isCreated(), equalTo(false)); + assertThat(r.getFailure(), instanceOf(UncheckedIOException.class)); + assertThat(r.getFailure().getCause(), instanceOf(MockDirectoryWrapper.FakeIOException.class)); + } } + // now the engine is closed check we respond correctly expectThrows(AlreadyClosedException.class, () -> engine.index(indexForDoc(doc1))); expectThrows(AlreadyClosedException.class, () -> engine.delete(new Engine.Delete("", newUid(doc1), primaryTerm.get()))); @@ -3986,7 +3990,7 @@ public ParsedDocument newDeleteTombstoneDoc(String id) { // this is a hack to add a failure during store document which triggers a tragic event // and in turn fails the engine @Override - public BytesRef binaryValue() { + public IndexableFieldType fieldType() { throw tragicException; } }); diff --git a/server/src/test/java/org/opensearch/index/engine/LuceneChangesSnapshotTests.java b/server/src/test/java/org/opensearch/index/engine/LuceneChangesSnapshotTests.java index d1a3097005e6c..b9b0e64d8811f 100644 --- a/server/src/test/java/org/opensearch/index/engine/LuceneChangesSnapshotTests.java +++ b/server/src/test/java/org/opensearch/index/engine/LuceneChangesSnapshotTests.java @@ -33,7 +33,7 @@ package org.opensearch.index.engine; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ParsedDocument; diff --git a/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java index e53a3b09a5eb6..b9fe69c282471 100644 --- a/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java @@ -46,7 +46,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.seqno.ReplicationTracker; diff --git a/server/src/test/java/org/opensearch/index/engine/ReadOnlyEngineTests.java b/server/src/test/java/org/opensearch/index/engine/ReadOnlyEngineTests.java index 2ce7c62cbdfbd..fed07e3805642 100644 --- a/server/src/test/java/org/opensearch/index/engine/ReadOnlyEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/ReadOnlyEngineTests.java @@ -42,7 +42,7 @@ import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ParsedDocument; diff --git a/server/src/test/java/org/opensearch/index/fielddata/plain/HalfFloatFielddataTests.java b/server/src/test/java/org/opensearch/index/fielddata/plain/HalfFloatFielddataTests.java index be631cc311b2a..62c475c88045a 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/plain/HalfFloatFielddataTests.java +++ b/server/src/test/java/org/opensearch/index/fielddata/plain/HalfFloatFielddataTests.java @@ -39,7 +39,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReader; import org.apache.lucene.store.Directory; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.apache.lucene.tests.util.TestUtil; import org.opensearch.index.fielddata.FieldData; import org.opensearch.index.fielddata.SortedNumericDoubleValues; diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java index 2300d3cceab65..d360a2a767e8a 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java @@ -52,7 +52,7 @@ import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.DateMathParser; import org.opensearch.common.util.BigArrays; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.fielddata.LeafNumericFieldData; diff --git a/server/src/test/java/org/opensearch/index/mapper/IgnoredFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/IgnoredFieldTypeTests.java index 4749c3280c89e..ef1530557d587 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IgnoredFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IgnoredFieldTypeTests.java @@ -33,11 +33,14 @@ package org.opensearch.index.mapper; import org.apache.lucene.index.Term; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; import org.opensearch.OpenSearchException; public class IgnoredFieldTypeTests extends FieldTypeTestCase { @@ -45,7 +48,7 @@ public class IgnoredFieldTypeTests extends FieldTypeTestCase { public void testPrefixQuery() { MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; - Query expected = new PrefixQuery(new Term("_ignored", new BytesRef("foo*"))); + Query expected = new PrefixQuery(new Term("_ignored", new BytesRef("foo*")), MultiTermQuery.CONSTANT_SCORE_REWRITE); assertEquals(expected, ft.prefixQuery("foo*", null, MOCK_QSC)); OpenSearchException ee = expectThrows(OpenSearchException.class, () -> ft.prefixQuery("foo*", null, MOCK_QSC_DISALLOW_EXPENSIVE)); @@ -59,7 +62,14 @@ public void testPrefixQuery() { public void testRegexpQuery() { MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; - Query expected = new RegexpQuery(new Term("_ignored", new BytesRef("foo?"))); + Query expected = new RegexpQuery( + new Term("_ignored", new BytesRef("foo?")), + RegExp.ALL, + 0, + RegexpQuery.DEFAULT_PROVIDER, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ); assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, null, MOCK_QSC)); OpenSearchException ee = expectThrows( @@ -73,7 +83,7 @@ public void testWildcardQuery() { MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; Query expected = new WildcardQuery(new Term("_ignored", new BytesRef("foo*"))); - assertEquals(expected, ft.wildcardQuery("foo*", null, MOCK_QSC)); + assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); OpenSearchException ee = expectThrows( OpenSearchException.class, diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java index ad529c685d6f3..626af63fc968d 100644 --- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java @@ -43,6 +43,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.NormsFieldExistsQuery; import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.TermInSetQuery; @@ -172,7 +173,10 @@ public void testRangeQuery() { public void testRegexpQuery() { MappedFieldType ft = new KeywordFieldType("field"); - assertEquals(new RegexpQuery(new Term("field", "foo.*")), ft.regexpQuery("foo.*", 0, 0, 10, null, MOCK_QSC)); + assertEquals( + new RegexpQuery(new Term("field", "foo.*")), + ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC) + ); MappedFieldType unsearchable = new KeywordFieldType("field", false, true, Collections.emptyMap()); IllegalArgumentException e = expectThrows( diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java index 863e29abaedf7..d43d009f57566 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java @@ -59,7 +59,7 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.MappedFieldType.Relation; diff --git a/server/src/test/java/org/opensearch/index/mapper/RoutingFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/RoutingFieldTypeTests.java index 8d3a8ea43f264..7553a6c621f62 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RoutingFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RoutingFieldTypeTests.java @@ -32,6 +32,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.index.Term; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; @@ -45,7 +46,7 @@ public void testPrefixQuery() { MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; Query expected = new PrefixQuery(new Term("_routing", new BytesRef("foo*"))); - assertEquals(expected, ft.prefixQuery("foo*", null, MOCK_QSC)); + assertEquals(expected, ft.prefixQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); OpenSearchException ee = expectThrows(OpenSearchException.class, () -> ft.prefixQuery("foo*", null, MOCK_QSC_DISALLOW_EXPENSIVE)); assertEquals( @@ -59,7 +60,7 @@ public void testRegexpQuery() { MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; Query expected = new RegexpQuery(new Term("_routing", new BytesRef("foo?"))); - assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, null, MOCK_QSC)); + assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); OpenSearchException ee = expectThrows( OpenSearchException.class, @@ -72,7 +73,7 @@ public void testWildcardQuery() { MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; Query expected = new WildcardQuery(new Term("_routing", new BytesRef("foo*"))); - assertEquals(expected, ft.wildcardQuery("foo*", null, MOCK_QSC)); + assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); OpenSearchException ee = expectThrows( OpenSearchException.class, diff --git a/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java index 206be8c8352ef..b8d470e6bdf3e 100644 --- a/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java @@ -61,6 +61,7 @@ import java.util.List; import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; +import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE; import static org.hamcrest.Matchers.equalTo; public class TextFieldTypeTests extends FieldTypeTestCase { @@ -120,7 +121,10 @@ public void testRangeQuery() { public void testRegexpQuery() { MappedFieldType ft = createFieldType(); - assertEquals(new RegexpQuery(new Term("field", "foo.*")), ft.regexpQuery("foo.*", 0, 0, 10, null, MOCK_QSC)); + assertEquals( + new RegexpQuery(new Term("field", "foo.*")), + ft.regexpQuery("foo.*", 0, 0, 10, CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC) + ); MappedFieldType unsearchable = new TextFieldType("field", false, false, Collections.emptyMap()); IllegalArgumentException e = expectThrows( @@ -165,7 +169,7 @@ public void testIndexPrefixes() { assertEquals(new ConstantScoreQuery(new TermQuery(new Term("field._index_prefix", "goin"))), q); q = ft.prefixQuery("internationalisatio", CONSTANT_SCORE_REWRITE, false, MOCK_QSC); - assertEquals(new PrefixQuery(new Term("field", "internationalisatio")), q); + assertEquals(new PrefixQuery(new Term("field", "internationalisatio"), CONSTANT_SCORE_REWRITE), q); q = ft.prefixQuery("Internationalisatio", CONSTANT_SCORE_REWRITE, true, MOCK_QSC); assertEquals(AutomatonQueries.caseInsensitivePrefixQuery(new Term("field", "Internationalisatio")), q); @@ -184,9 +188,16 @@ public void testIndexPrefixes() { Automaton automaton = Operations.concatenate(Arrays.asList(Automata.makeChar('g'), Automata.makeAnyChar())); Query expected = new ConstantScoreQuery( - new BooleanQuery.Builder().add(new AutomatonQuery(new Term("field._index_prefix", "g*"), automaton), BooleanClause.Occur.SHOULD) - .add(new TermQuery(new Term("field", "g")), BooleanClause.Occur.SHOULD) - .build() + new BooleanQuery.Builder().add( + new AutomatonQuery( + new Term("field._index_prefix", "g*"), + automaton, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + false, + CONSTANT_SCORE_REWRITE + ), + BooleanClause.Occur.SHOULD + ).add(new TermQuery(new Term("field", "g")), BooleanClause.Occur.SHOULD).build() ); assertThat(q, equalTo(expected)); @@ -195,9 +206,16 @@ public void testIndexPrefixes() { automaton = Operations.concatenate(Arrays.asList(Automata.makeChar('g'), Automata.makeAnyChar())); expected = new ConstantScoreQuery( - new BooleanQuery.Builder().add(new AutomatonQuery(new Term("field._index_prefix", "g*"), automaton), BooleanClause.Occur.SHOULD) - .add(new TermQuery(new Term("field", "g")), BooleanClause.Occur.SHOULD) - .build() + new BooleanQuery.Builder().add( + new AutomatonQuery( + new Term("field._index_prefix", "g*"), + automaton, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + false, + CONSTANT_SCORE_REWRITE + ), + BooleanClause.Occur.SHOULD + ).add(new TermQuery(new Term("field", "g")), BooleanClause.Occur.SHOULD).build() ); assertThat(q, equalTo(expected)); diff --git a/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java index bc8cd6c57b975..8d589bd76f2bb 100644 --- a/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java @@ -35,6 +35,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.opensearch.test.AbstractQueryTestCase; @@ -103,7 +104,10 @@ public void testToQueryInnerPrefixQuery() throws Exception { + " }\n" + "}"; Query query = parseQuery(queryAsString).toQuery(createShardContext()); - Query expected = new DisjunctionMaxQuery(List.of(new BoostQuery(new PrefixQuery(new Term(TEXT_FIELD_NAME, "sh")), 1.2f)), 0); + Query expected = new DisjunctionMaxQuery( + List.of(new BoostQuery(new PrefixQuery(new Term(TEXT_FIELD_NAME, "sh"), MultiTermQuery.CONSTANT_SCORE_REWRITE), 1.2f)), + 0 + ); assertEquals(expected, query); } diff --git a/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java index 0e3fd0efe7804..a23b4fd75e8a0 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java @@ -37,6 +37,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; @@ -268,7 +269,7 @@ public void testAnalysis() throws Exception { asList( new TermQuery(new Term(TEXT_FIELD_NAME, "foo")), new TermQuery(new Term(TEXT_FIELD_NAME, "bar")), - new PrefixQuery(new Term(TEXT_FIELD_NAME, "baz")) + new PrefixQuery(new Term(TEXT_FIELD_NAME, "baz"), MultiTermQuery.CONSTANT_SCORE_REWRITE) ) ); } @@ -285,7 +286,7 @@ public void testAnalysisSynonym() throws Exception { new SynonymQuery.Builder(TEXT_FIELD_NAME).addTerm(new Term(TEXT_FIELD_NAME, "dogs")) .addTerm(new Term(TEXT_FIELD_NAME, "dog")) .build(), - new PrefixQuery(new Term(TEXT_FIELD_NAME, "red")) + new PrefixQuery(new Term(TEXT_FIELD_NAME, "red"), MultiTermQuery.CONSTANT_SCORE_REWRITE) ) ); } @@ -293,7 +294,7 @@ public void testAnalysisSynonym() throws Exception { public void testAnalysisSingleTerm() throws Exception { final MatchBoolPrefixQueryBuilder builder = new MatchBoolPrefixQueryBuilder(TEXT_FIELD_NAME, "foo"); final Query query = builder.toQuery(createShardContext()); - assertThat(query, equalTo(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foo")))); + assertThat(query, equalTo(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foo"), MultiTermQuery.CONSTANT_SCORE_REWRITE))); } private static void assertBooleanQuery(Query actual, List expectedClauseQueries) { diff --git a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java index 40fac046dfbb9..03ddb625c78a8 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java @@ -567,7 +567,12 @@ public void testToQueryWildcardQuery() throws Exception { .add( new BooleanQuery.Builder().add(new BooleanClause(new TermQuery(new Term(TEXT_FIELD_NAME, "foo")), defaultOp)) .add(new BooleanClause(new TermQuery(new Term(TEXT_FIELD_NAME, "bar")), defaultOp)) - .add(new BooleanClause(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foobar")), defaultOp)) + .add( + new BooleanClause( + new PrefixQuery(new Term(TEXT_FIELD_NAME, "foobar"), MultiTermQuery.CONSTANT_SCORE_REWRITE), + defaultOp + ) + ) .build(), defaultOp ) @@ -579,6 +584,8 @@ public void testToQueryWildcardQuery() throws Exception { public void testToQueryWildcardWithIndexedPrefixes() throws Exception { QueryStringQueryParser queryParser = new QueryStringQueryParser(createShardContext(), "prefix_field"); + queryParser.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE); + Query query = queryParser.parse("foo*"); Query expectedQuery = new ConstantScoreQuery(new TermQuery(new Term("prefix_field._index_prefix", "foo"))); assertThat(query, equalTo(expectedQuery)); @@ -586,9 +593,16 @@ public void testToQueryWildcardWithIndexedPrefixes() throws Exception { query = queryParser.parse("g*"); Automaton a = Operations.concatenate(Arrays.asList(Automata.makeChar('g'), Automata.makeAnyChar())); expectedQuery = new ConstantScoreQuery( - new BooleanQuery.Builder().add(new AutomatonQuery(new Term("prefix_field._index_prefix", "g*"), a), Occur.SHOULD) - .add(new TermQuery(new Term("prefix_field", "g")), Occur.SHOULD) - .build() + new BooleanQuery.Builder().add( + new AutomatonQuery( + new Term("prefix_field._index_prefix", "g*"), + a, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + false, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + Occur.SHOULD + ).add(new TermQuery(new Term("prefix_field", "g")), Occur.SHOULD).build() ); assertThat(query, equalTo(expectedQuery)); } @@ -598,7 +612,7 @@ public void testToQueryWildcardQueryWithSynonyms() throws Exception { BooleanClause.Occur defaultOp = op.toBooleanClauseOccur(); QueryStringQueryParser queryParser = new QueryStringQueryParser(createShardContext(), TEXT_FIELD_NAME); queryParser.setAnalyzeWildcard(true); - queryParser.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE); + queryParser.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE); queryParser.setDefaultOperator(op.toQueryParserOperator()); queryParser.setForceAnalyzer(new MockRepeatAnalyzer()); Query query = queryParser.parse("first foo-bar-foobar* last"); @@ -1157,15 +1171,29 @@ public void testFromJson() throws IOException { public void testExpandedTerms() throws Exception { // Prefix Query query = new QueryStringQueryBuilder("aBc*").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); - assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "aBc")), query); + assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "aBc"), MultiTermQuery.CONSTANT_SCORE_REWRITE), query); query = new QueryStringQueryBuilder("aBc*").field(TEXT_FIELD_NAME).analyzer("standard").toQuery(createShardContext()); - assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "abc")), query); + assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "abc"), MultiTermQuery.CONSTANT_SCORE_REWRITE), query); // Wildcard query = new QueryStringQueryBuilder("aBc*D").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); - assertEquals(new WildcardQuery(new Term(TEXT_FIELD_NAME, "aBc*D")), query); + assertEquals( + new WildcardQuery( + new Term(TEXT_FIELD_NAME, "aBc*D"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + query + ); query = new QueryStringQueryBuilder("aBc*D").field(TEXT_FIELD_NAME).analyzer("standard").toQuery(createShardContext()); - assertEquals(new WildcardQuery(new Term(TEXT_FIELD_NAME, "abc*d")), query); + assertEquals( + new WildcardQuery( + new Term(TEXT_FIELD_NAME, "abc*d"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + query + ); // Fuzzy query = new QueryStringQueryBuilder("aBc~1").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); @@ -1407,7 +1435,10 @@ public void testEnablePositionIncrement() throws Exception { public void testWithPrefixStopWords() throws Exception { Query query = new QueryStringQueryBuilder("the* quick fox").field(TEXT_FIELD_NAME).analyzer("stop").toQuery(createShardContext()); - BooleanQuery expected = new BooleanQuery.Builder().add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "the")), Occur.SHOULD) + BooleanQuery expected = new BooleanQuery.Builder().add( + new PrefixQuery(new Term(TEXT_FIELD_NAME, "the"), MultiTermQuery.CONSTANT_SCORE_REWRITE), + Occur.SHOULD + ) .add(new TermQuery(new Term(TEXT_FIELD_NAME, "quick")), Occur.SHOULD) .add(new TermQuery(new Term(TEXT_FIELD_NAME, "fox")), Occur.SHOULD) .build(); @@ -1488,7 +1519,7 @@ public void testAnalyzedPrefix() throws Exception { .analyzer("standard") .analyzeWildcard(true) .toQuery(createShardContext()); - Query expected = new PrefixQuery(new Term(TEXT_FIELD_NAME, "quick")); + Query expected = new PrefixQuery(new Term(TEXT_FIELD_NAME, "quick"), MultiTermQuery.CONSTANT_SCORE_REWRITE); assertEquals(expected, query); } diff --git a/server/src/test/java/org/opensearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SimpleQueryStringBuilderTests.java index fa8646d8628a0..7e86edb1edefe 100644 --- a/server/src/test/java/org/opensearch/index/query/SimpleQueryStringBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SimpleQueryStringBuilderTests.java @@ -45,6 +45,7 @@ import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; @@ -451,9 +452,9 @@ public void testMinimumShouldMatch() throws IOException { public void testExpandedTerms() throws Exception { // Prefix Query query = new SimpleQueryStringBuilder("aBc*").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); - assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "aBc")), query); + assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "aBc"), MultiTermQuery.CONSTANT_SCORE_REWRITE), query); query = new SimpleQueryStringBuilder("aBc*").field(TEXT_FIELD_NAME).analyzer("standard").toQuery(createShardContext()); - assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "abc")), query); + assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "abc"), MultiTermQuery.CONSTANT_SCORE_REWRITE), query); // Fuzzy query = new SimpleQueryStringBuilder("aBc~1").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); @@ -712,8 +713,12 @@ public void testLenientToPrefixQuery() throws Exception { .lenient(true) .toQuery(createShardContext()); List expectedQueries = new ArrayList<>(); - expectedQueries.add(new MatchNoDocsQuery("")); - expectedQueries.add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "t"))); + expectedQueries.add( + new MatchNoDocsQuery( + "failed query, caused by Can only use prefix queries on keyword and text fields - not on [mapped_date] which is of type [date]" + ) + ); + expectedQueries.add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "t"), MultiTermQuery.CONSTANT_SCORE_REWRITE)); DisjunctionMaxQuery expected = new DisjunctionMaxQuery(expectedQueries, 1.0f); assertEquals(expected, query); } @@ -766,7 +771,7 @@ public void testWithStopWords() throws Exception { public void testWithPrefixStopWords() throws Exception { Query query = new SimpleQueryStringBuilder("the* quick fox").field(TEXT_FIELD_NAME).analyzer("stop").toQuery(createShardContext()); BooleanQuery expected = new BooleanQuery.Builder().add( - new PrefixQuery(new Term(TEXT_FIELD_NAME, "the")), + new PrefixQuery(new Term(TEXT_FIELD_NAME, "the"), MultiTermQuery.CONSTANT_SCORE_REWRITE), BooleanClause.Occur.SHOULD ) .add(new TermQuery(new Term(TEXT_FIELD_NAME, "quick")), BooleanClause.Occur.SHOULD) diff --git a/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java index 75063d76ff8dc..241059867f5fd 100644 --- a/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java index 2e058b6dab560..b65662a53ea91 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.gateway.WriteStateException; import org.opensearch.index.Index; import org.opensearch.index.IndexService; diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java index b07b740fe3744..579b8ce1c5aa6 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.IndexingPressureService; diff --git a/server/src/test/java/org/opensearch/index/shard/GlobalCheckpointListenersTests.java b/server/src/test/java/org/opensearch/index/shard/GlobalCheckpointListenersTests.java index 1b7bdaf710de4..aa86b4067d566 100644 --- a/server/src/test/java/org/opensearch/index/shard/GlobalCheckpointListenersTests.java +++ b/server/src/test/java/org/opensearch/index/shard/GlobalCheckpointListenersTests.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; diff --git a/server/src/test/java/org/opensearch/index/shard/IndexReaderWrapperTests.java b/server/src/test/java/org/opensearch/index/shard/IndexReaderWrapperTests.java index 6b58633d5cd2e..f030c3cb21c62 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexReaderWrapperTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexReaderWrapperTests.java @@ -48,7 +48,7 @@ import org.apache.lucene.store.Directory; import org.opensearch.common.CheckedFunction; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.engine.Engine; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardOperationPermitsTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardOperationPermitsTests.java index 6e130c1938b4f..1e09ae56f0049 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardOperationPermitsTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardOperationPermitsTests.java @@ -36,7 +36,7 @@ import org.opensearch.common.CheckedRunnable; import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 9fc360e182439..563b1cf566bf4 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -49,7 +49,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; import org.junit.Assert; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionListener; diff --git a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java index dbb0207b4f1b7..9bb36bc32639a 100644 --- a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java @@ -52,7 +52,7 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecService; diff --git a/server/src/test/java/org/opensearch/index/shard/ShardUtilsTests.java b/server/src/test/java/org/opensearch/index/shard/ShardUtilsTests.java index 25a385f157b5c..fa30fbbac9a0d 100644 --- a/server/src/test/java/org/opensearch/index/shard/ShardUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ShardUtilsTests.java @@ -39,7 +39,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.tests.store.BaseDirectoryWrapper; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java b/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java index 64b6490ace1c7..1f384285bd633 100644 --- a/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/StoreRecoveryTests.java @@ -56,7 +56,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.OperationRouting; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.engine.Engine; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.Uid; diff --git a/server/src/test/java/org/opensearch/index/store/StoreTests.java b/server/src/test/java/org/opensearch/index/store/StoreTests.java index 3bcc244bf7fc5..b810aa7d197f2 100644 --- a/server/src/test/java/org/opensearch/index/store/StoreTests.java +++ b/server/src/test/java/org/opensearch/index/store/StoreTests.java @@ -75,7 +75,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.ShardLock; import org.opensearch.index.Index; import org.opensearch.index.IndexModule; diff --git a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java index 8d19c265da052..6c1741d68cb05 100644 --- a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java @@ -49,7 +49,7 @@ import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.apache.lucene.tests.util.LineFileDocs; import org.apache.lucene.tests.util.LuceneTestCase; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Randomness; @@ -72,7 +72,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.engine.Engine; diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java index d5acf3efde6e6..0a6b6a95b74f9 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java @@ -36,7 +36,7 @@ import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; import org.opensearch.index.IndexSettings; diff --git a/server/src/test/java/org/opensearch/index/translog/TestTranslog.java b/server/src/test/java/org/opensearch/index/translog/TestTranslog.java index 3545f2dbf5ffc..73a3a1c646d5b 100644 --- a/server/src/test/java/org/opensearch/index/translog/TestTranslog.java +++ b/server/src/test/java/org/opensearch/index/translog/TestTranslog.java @@ -37,7 +37,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java index e5a41fdc15da1..983a42880e34c 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java @@ -38,7 +38,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.util.BigArrays; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.mockito.Mockito; diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java index 4c3948cbd7b5b..44e8d52b034de 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java @@ -24,7 +24,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.Engine; diff --git a/server/src/test/java/org/opensearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesQueryCacheTests.java index 24d8d51042548..b1a44d94aa0e1 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesQueryCacheTests.java @@ -51,7 +51,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.settings.Settings; import org.opensearch.index.cache.query.QueryCacheStats; import org.opensearch.index.shard.ShardId; diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java index 8342524b264aa..d10b99f42e0df 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java @@ -54,7 +54,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.cache.request.ShardRequestCache; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.shard.ShardId; diff --git a/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java index 307cb854f000a..ae59798d0695f 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java @@ -72,7 +72,7 @@ import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.RecoveryEngineException; diff --git a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java index ff4058f953642..991215711f039 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java @@ -48,7 +48,7 @@ import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.NoOpEngine; import org.opensearch.index.mapper.SourceToParse; diff --git a/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java index d925956bd95ef..10b747b822819 100644 --- a/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java @@ -17,7 +17,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.CancellableThreads; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.Store; diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentFileTransferHandlerTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentFileTransferHandlerTests.java index 0400d73da4503..8fc7b8b6f4908 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentFileTransferHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentFileTransferHandlerTests.java @@ -16,7 +16,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.util.CancellableThreads; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.Store; diff --git a/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java b/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java index 2c05fbc9328e5..8a67292703da0 100644 --- a/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java @@ -17,7 +17,7 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index 81a4a4eace608..63b144dae9c93 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -45,7 +45,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; import org.opensearch.index.engine.EngineConfigFactory; diff --git a/server/src/test/java/org/opensearch/rest/RestControllerTests.java b/server/src/test/java/org/opensearch/rest/RestControllerTests.java index 57ea5e2928cb3..ace7762989411 100644 --- a/server/src/test/java/org/opensearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/opensearch/rest/RestControllerTests.java @@ -46,7 +46,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.yaml.YamlXContent; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.http.HttpInfo; import org.opensearch.http.HttpRequest; import org.opensearch.http.HttpResponse; diff --git a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java index 96a4d9ad1d8d9..d2242b7d3f07e 100644 --- a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java @@ -52,7 +52,7 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.cache.IndexCache; diff --git a/server/src/test/java/org/opensearch/search/SearchCancellationTests.java b/server/src/test/java/org/opensearch/search/SearchCancellationTests.java index f479f3a1b99f1..db807e31d0238 100644 --- a/server/src/test/java/org/opensearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/opensearch/search/SearchCancellationTests.java @@ -48,7 +48,7 @@ import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.RegExp; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.search.internal.ContextIndexSearcher; import org.opensearch.tasks.TaskCancelledException; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java index a0747d5ddff14..4540e860b23ef 100644 --- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java @@ -61,7 +61,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.Index; import org.opensearch.index.IndexModule; diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index eb7dde4b0b2ce..82dcbd6f69aa4 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -77,7 +77,7 @@ import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.shard.ShardId; diff --git a/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java index 7f4dcdaed2aa1..da2a70514a716 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java @@ -58,7 +58,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.TestUtil; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.search.internal.ContextIndexSearcher; import org.opensearch.search.profile.ProfileResult; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/threadpool/FixedThreadPoolTests.java b/server/src/test/java/org/opensearch/threadpool/FixedThreadPoolTests.java index 52b4a4ce6a231..3cba803e84e68 100644 --- a/server/src/test/java/org/opensearch/threadpool/FixedThreadPoolTests.java +++ b/server/src/test/java/org/opensearch/threadpool/FixedThreadPoolTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.threadpool.ThreadPool.Names; import java.util.concurrent.CountDownLatch; diff --git a/server/src/test/java/org/opensearch/threadpool/ScheduleWithFixedDelayTests.java b/server/src/test/java/org/opensearch/threadpool/ScheduleWithFixedDelayTests.java index 0e5dc23bf8b5c..cfa0b7738e2ac 100644 --- a/server/src/test/java/org/opensearch/threadpool/ScheduleWithFixedDelayTests.java +++ b/server/src/test/java/org/opensearch/threadpool/ScheduleWithFixedDelayTests.java @@ -36,7 +36,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.BaseFuture; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.node.Node; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.Scheduler.Cancellable; diff --git a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java index 43de41bff69d4..7be191e00d968 100644 --- a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java @@ -47,7 +47,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.Streams; +import org.opensearch.common.util.io.Streams; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index c4a25f7d80be7..1cc115bf65a00 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -48,7 +48,7 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.internal.io.Streams; +import org.opensearch.common.util.io.Streams; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java index fdbb3280e4278..86b40b3f84536 100644 --- a/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java @@ -60,7 +60,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexNotFoundException; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/server/src/test/java/org/opensearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/opensearch/transport/RemoteClusterServiceTests.java index b8dd17da45acb..dea42684d2e21 100644 --- a/server/src/test/java/org/opensearch/transport/RemoteClusterServiceTests.java +++ b/server/src/test/java/org/opensearch/transport/RemoteClusterServiceTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java index 0488f249cb09e..2bbbab1b2ccbd 100644 --- a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java @@ -38,7 +38,7 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/watcher/FileWatcherTests.java b/server/src/test/java/org/opensearch/watcher/FileWatcherTests.java index 74c9cd91b72f0..3e90784a15ab9 100644 --- a/server/src/test/java/org/opensearch/watcher/FileWatcherTests.java +++ b/server/src/test/java/org/opensearch/watcher/FileWatcherTests.java @@ -31,7 +31,7 @@ package org.opensearch.watcher; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.test.OpenSearchTestCase; diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java index 24e5af1ba960f..9370a2a211725 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java @@ -87,7 +87,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.MapperTestUtils; diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index f21cc44d94243..7b81bd45cc8b6 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -73,7 +73,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.Index; import org.opensearch.index.IndexSettings; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java index 3eb56c9451c97..99659e06102b3 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java @@ -57,7 +57,7 @@ import org.opensearch.common.settings.SettingsModule; import org.opensearch.common.util.BigArrays; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; import org.opensearch.index.Index; diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractMultiClustersTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractMultiClustersTestCase.java index 69d30beaa8e16..f50603a7d8e1e 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractMultiClustersTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractMultiClustersTestCase.java @@ -37,7 +37,7 @@ import org.opensearch.client.Client; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.plugins.Plugin; import org.opensearch.test.transport.MockTransportService; import org.opensearch.transport.RemoteClusterAware; diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index f9ae36a1a55e6..121003f66fc94 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -87,7 +87,7 @@ import org.opensearch.common.util.concurrent.FutureUtils; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLockObtainFailedException; diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 7c074b07f5f3d..2cdcd70869069 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -110,7 +110,7 @@ import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -119,7 +119,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.smile.SmileXContent; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; import org.opensearch.http.HttpInfo; diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java index c8f475ba96edd..3d03c3d4b04c9 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java @@ -52,7 +52,7 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.Index; diff --git a/test/framework/src/main/java/org/opensearch/test/StreamsUtils.java b/test/framework/src/main/java/org/opensearch/test/StreamsUtils.java index 6579815deeb48..644274b135b7e 100644 --- a/test/framework/src/main/java/org/opensearch/test/StreamsUtils.java +++ b/test/framework/src/main/java/org/opensearch/test/StreamsUtils.java @@ -34,7 +34,7 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.internal.io.Streams; +import org.opensearch.common.util.io.Streams; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java index 70dfba204ff9e..a5d2e6793f397 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java @@ -73,7 +73,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.support.XContentMapValues; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.rest.RestStatus; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/OpenSearchClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/OpenSearchClientYamlSuiteTestCase.java index d9771e86ccba3..43482cf0f6656 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/OpenSearchClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/OpenSearchClientYamlSuiteTestCase.java @@ -51,7 +51,7 @@ import org.opensearch.common.io.PathUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.test.rest.OpenSearchRestTestCase; import org.opensearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi; import org.opensearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec; diff --git a/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java index 0324ef34c4958..cff363cabf5a3 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java @@ -52,7 +52,7 @@ import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.RunOnce; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.indices.breaker.NoneCircuitBreakerService; import org.opensearch.node.Node; import org.opensearch.plugins.Plugin; diff --git a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java index 106d2ae15917f..fed14d6bbc1ba 100644 --- a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java @@ -63,7 +63,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.node.Node; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; diff --git a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java index 261097a9d648a..a1f7f5fd2c901 100644 --- a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java +++ b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java @@ -52,7 +52,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.nio.BytesChannelContext; import org.opensearch.nio.BytesWriteHandler; diff --git a/test/framework/src/test/java/org/opensearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/opensearch/test/test/InternalTestClusterTests.java index b802b48960f61..d68cedac29674 100644 --- a/test/framework/src/test/java/org/opensearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/opensearch/test/test/InternalTestClusterTests.java @@ -38,7 +38,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.discovery.DiscoveryModule; import org.opensearch.discovery.SettingsBasedSeedHostsProvider; import org.opensearch.env.Environment;