-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor Compressors from CompressorFactory to CompressorRegistry for…
… extensibility (#9262) (#9367) This commit refactors the CompressorFactory static singleton class and CompressorType enum to a formal CompressorRegistry and enables downstream implementations to register their own compression implementations for use in compressing Blob stores and MediaType data. This is different from Lucene's Codec compression extension points which expose different compression implementations for Lucene's Stored Fields. --------- Signed-off-by: Nicholas Walter Knize <[email protected]> (cherry picked from commit c6b67e1)
- Loading branch information
Showing
49 changed files
with
526 additions
and
244 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| * | ||
| * Modifications Copyright OpenSearch Contributors. See | ||
| * GitHub history for details. | ||
| */ | ||
|
|
||
| apply plugin: 'opensearch.build' | ||
| apply plugin: 'opensearch.publish' | ||
|
|
||
| base { | ||
| archivesName = 'opensearch-compress' | ||
| } | ||
|
|
||
| dependencies { | ||
| api project(':libs:opensearch-common') | ||
| api project(':libs:opensearch-core') | ||
|
|
||
| //zstd | ||
| api "com.github.luben:zstd-jni:${versions.zstd}" | ||
|
|
||
| testImplementation(project(":test:framework")) { | ||
| // tests use the locally compiled version of server | ||
| exclude group: 'org.opensearch', module: 'opensearch-compress' | ||
| } | ||
| } | ||
|
|
||
| tasks.named('forbiddenApisMain').configure { | ||
| // :libs:opensearch-compress does not depend on server | ||
| // TODO: Need to decide how we want to handle for forbidden signatures with the changes to server | ||
| replaceSignatureFiles 'jdk-signatures' | ||
| } | ||
|
|
||
| jarHell.enabled = false |
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
32 changes: 32 additions & 0 deletions
32
libs/compress/src/main/java/org/opensearch/compress/spi/CompressionProvider.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.compress.spi; | ||
|
|
||
| import org.opensearch.compress.ZstdCompressor; | ||
| import org.opensearch.core.compress.Compressor; | ||
| import org.opensearch.core.compress.spi.CompressorProvider; | ||
|
|
||
| import java.util.AbstractMap.SimpleEntry; | ||
| import java.util.List; | ||
| import java.util.Map.Entry; | ||
|
|
||
| /** | ||
| * Additional "optional" compressor implementations provided by the opensearch compress library | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public class CompressionProvider implements CompressorProvider { | ||
|
|
||
| /** Returns the concrete {@link Compressor}s provided by the compress library */ | ||
| @SuppressWarnings({ "unchecked", "rawtypes" }) | ||
| @Override | ||
| public List<Entry<String, Compressor>> getCompressors() { | ||
| return List.of(new SimpleEntry<>(ZstdCompressor.NAME, new ZstdCompressor())); | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
libs/compress/src/main/java/org/opensearch/compress/spi/package-info.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| /** | ||
| * Service Provider Interface for registering concrete {@link org.opensearch.core.compress.Compressor} | ||
| * implementations. | ||
| * | ||
| * See {@link org.opensearch.compress.ZstdCompressor} | ||
| */ | ||
| package org.opensearch.compress.spi; |
13 changes: 13 additions & 0 deletions
13
libs/compress/src/main/java/org/opensearch/package-info.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| /** | ||
| * This is the compress library for registering optional | ||
| * {@link org.opensearch.core.compress.Compressor} implementations | ||
| */ | ||
| package org.opensearch; |
9 changes: 9 additions & 0 deletions
9
.../src/main/resources/META-INF/services/org.opensearch.core.compress.spi.CompressorProvider
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| # | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # | ||
| # The OpenSearch Contributors require contributions made to | ||
| # this file be licensed under the Apache-2.0 license or a | ||
| # compatible open source license. | ||
| # | ||
|
|
||
| org.opensearch.compress.spi.CompressionProvider |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
115 changes: 115 additions & 0 deletions
115
libs/core/src/main/java/org/opensearch/core/compress/CompressorRegistry.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.core.compress; | ||
|
|
||
| import org.opensearch.common.Nullable; | ||
| import org.opensearch.common.annotation.InternalApi; | ||
| import org.opensearch.core.common.bytes.BytesReference; | ||
| import org.opensearch.core.compress.spi.CompressorProvider; | ||
| import org.opensearch.core.xcontent.MediaTypeRegistry; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.ServiceLoader; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| /** | ||
| * A registry that wraps a static Map singleton which holds a mapping of unique String names (typically the | ||
| * compressor header as a string) to registerd {@link Compressor} implementations. | ||
| * | ||
| * This enables plugins, modules, extensions to register their own compression implementations through SPI | ||
| * | ||
| * @opensearch.experimental | ||
| * @opensearch.internal | ||
| */ | ||
| @InternalApi | ||
| public final class CompressorRegistry { | ||
|
|
||
| // the backing registry map | ||
| private static final Map<String, Compressor> registeredCompressors = ServiceLoader.load( | ||
| CompressorProvider.class, | ||
| CompressorProvider.class.getClassLoader() | ||
| ) | ||
| .stream() | ||
| .flatMap(p -> p.get().getCompressors().stream()) | ||
| .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); | ||
|
|
||
| // no instance: | ||
| private CompressorRegistry() {} | ||
|
|
||
| /** | ||
| * Returns the default compressor | ||
| */ | ||
| public static Compressor defaultCompressor() { | ||
| return registeredCompressors.get("DEFLATE"); | ||
| } | ||
|
|
||
| public static Compressor none() { | ||
| return registeredCompressors.get(NoneCompressor.NAME); | ||
| } | ||
|
|
||
| public static boolean isCompressed(BytesReference bytes) { | ||
| return compressor(bytes) != null; | ||
| } | ||
|
|
||
| @Nullable | ||
| public static Compressor compressor(final BytesReference bytes) { | ||
| for (Compressor compressor : registeredCompressors.values()) { | ||
| if (compressor.isCompressed(bytes) == true) { | ||
| // bytes should be either detected as compressed or as xcontent, | ||
| // if we have bytes that can be either detected as compressed or | ||
| // as a xcontent, we have a problem | ||
| assert MediaTypeRegistry.xContentType(bytes) == null; | ||
| return compressor; | ||
| } | ||
| } | ||
|
|
||
| if (MediaTypeRegistry.xContentType(bytes) == null) { | ||
| throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); | ||
| } | ||
|
|
||
| return null; | ||
| } | ||
|
|
||
| /** Decompress the provided {@link BytesReference}. */ | ||
| public static BytesReference uncompress(BytesReference bytes) throws IOException { | ||
| Compressor compressor = compressor(bytes); | ||
| if (compressor == null) { | ||
| throw new NotCompressedException(); | ||
| } | ||
| return compressor.uncompress(bytes); | ||
| } | ||
|
|
||
| /** | ||
| * Uncompress the provided data, data can be detected as compressed using {@link #isCompressed(BytesReference)}. | ||
| */ | ||
| public static BytesReference uncompressIfNeeded(BytesReference bytes) throws IOException { | ||
| Compressor compressor = compressor(Objects.requireNonNull(bytes, "the BytesReference must not be null")); | ||
| return compressor == null ? bytes : compressor.uncompress(bytes); | ||
| } | ||
|
|
||
| /** Returns a registered compressor by its registered name */ | ||
| public static Compressor getCompressor(final String name) { | ||
| if (registeredCompressors.containsKey(name)) { | ||
| return registeredCompressors.get(name); | ||
| } | ||
| throw new IllegalArgumentException("No registered compressor found by name [" + name + "]"); | ||
| } | ||
|
|
||
| /** | ||
| * Returns the registered compressors as an Immutable collection | ||
| * | ||
| * note: used for testing | ||
| */ | ||
| public static Map<String, Compressor> registeredCompressors() { | ||
| // no destructive danger as backing map is immutable | ||
| return registeredCompressors; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.