-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
introducing zstd compression codec plugin
Signed-off-by: Sarthak Aggarwal <[email protected]>
- Loading branch information
1 parent
79e5aee
commit 0d0aea6
Showing
29 changed files
with
518 additions
and
114 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| * | ||
| * Modifications Copyright OpenSearch Contributors. See | ||
| * GitHub history for details. | ||
| */ | ||
|
|
||
| apply plugin: 'opensearch.opensearchplugin' | ||
| apply plugin: 'opensearch.yaml-rest-test' | ||
| apply plugin: 'opensearch.internal-cluster-test' | ||
|
|
||
| opensearchplugin { | ||
| name 'custom-codecs' | ||
| description 'A plugin that implements custom compression codecs.' | ||
| classname 'org.opensearch.index.codec.customcodecs.CustomCodecPlugin' | ||
| licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') | ||
| noticeFile rootProject.file('NOTICE.txt') | ||
| } | ||
|
|
||
| dependencies { | ||
| api "com.github.luben:zstd-jni:1.5.5-5" | ||
| } | ||
|
|
||
| yamlRestTest.enabled = false; | ||
| testingConventions.enabled = false; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
47 changes: 47 additions & 0 deletions
47
...ustom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.opensearch.index.IndexSettings; | ||
| import org.opensearch.index.codec.CodecServiceFactory; | ||
| import org.opensearch.index.engine.EngineConfig; | ||
| import org.opensearch.plugins.EnginePlugin; | ||
| import org.opensearch.plugins.Plugin; | ||
|
|
||
| import java.util.Optional; | ||
|
|
||
| /** | ||
| * A plugin that implements custom codecs. Supports these codecs: | ||
| * <ul> | ||
| * <li>ZSTD | ||
| * <li>ZSTDNODICT | ||
| * </ul> | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public final class CustomCodecPlugin extends Plugin implements EnginePlugin { | ||
|
|
||
| /** | ||
| * Creates a new instance | ||
| */ | ||
| public CustomCodecPlugin() {} | ||
|
|
||
| /** | ||
| * @param indexSettings is the default indexSettings | ||
| * @return the engine factory | ||
| */ | ||
| @Override | ||
| public Optional<CodecServiceFactory> getCustomCodecServiceFactory(final IndexSettings indexSettings) { | ||
| String codecName = indexSettings.getValue(EngineConfig.INDEX_CODEC_SETTING); | ||
| if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) || codecName.equals(CustomCodecService.ZSTD_CODEC)) { | ||
| return Optional.of(new CustomCodecServiceFactory()); | ||
| } | ||
| return Optional.empty(); | ||
| } | ||
| } |
74 changes: 74 additions & 0 deletions
74
...stom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.lucene.codecs.Codec; | ||
| import org.opensearch.common.collect.MapBuilder; | ||
| import org.opensearch.index.IndexSettings; | ||
| import org.opensearch.index.codec.CodecService; | ||
| import org.opensearch.index.mapper.MapperService; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.Arrays; | ||
| import java.util.Map; | ||
|
|
||
| import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; | ||
|
|
||
| /** | ||
| * CustomCodecService provides ZSTD and ZSTD_NO_DICT compression codecs. | ||
| */ | ||
| public class CustomCodecService extends CodecService { | ||
| private final Map<String, Codec> codecs; | ||
| /** | ||
| * ZStandard codec | ||
| */ | ||
| public static final String ZSTD_CODEC = "zstd"; | ||
| /** | ||
| * ZStandard without dictionary codec | ||
| */ | ||
| public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; | ||
|
|
||
| /** | ||
| * Creates a new CustomCodecService. | ||
| * | ||
| * @param mapperService The mapper service. | ||
| * @param indexSettings The index settings. | ||
| * @param logger The logger. | ||
| */ | ||
| public CustomCodecService(MapperService mapperService, IndexSettings indexSettings, Logger logger) { | ||
| super(mapperService, indexSettings, logger); | ||
| int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); | ||
| final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder(); | ||
| if (mapperService == null) { | ||
| codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); | ||
| codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); | ||
| } else { | ||
| codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); | ||
| codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); | ||
| } | ||
| this.codecs = codecs.immutableMap(); | ||
| } | ||
|
|
||
| @Override | ||
| public Codec codec(String name) { | ||
| Codec codec = codecs.get(name); | ||
| if (codec == null) { | ||
| return super.codec(name); | ||
| } | ||
| return codec; | ||
| } | ||
|
|
||
| @Override | ||
| public String[] availableCodecs() { | ||
| ArrayList<String> ac = new ArrayList<String>(Arrays.asList(super.availableCodecs())); | ||
| ac.addAll(codecs.keySet()); | ||
| return ac.toArray(new String[0]); | ||
| } | ||
| } |
27 changes: 27 additions & 0 deletions
27
...decs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.opensearch.index.codec.CodecService; | ||
| import org.opensearch.index.codec.CodecServiceConfig; | ||
| import org.opensearch.index.codec.CodecServiceFactory; | ||
|
|
||
| /** | ||
| * A factory for creating new {@link CodecService} instance | ||
| */ | ||
| public class CustomCodecServiceFactory implements CodecServiceFactory { | ||
|
|
||
| /** Creates a new instance. */ | ||
| public CustomCodecServiceFactory() {} | ||
|
|
||
| @Override | ||
| public CodecService createCodecService(CodecServiceConfig config) { | ||
| return new CustomCodecService(config.getMapperService(), config.getIndexSettings(), config.getLogger()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 25 additions & 0 deletions
25
.../main/java/org/opensearch/index/codec/customcodecs/PerFieldMappingPostingFormatCodec.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.opensearch.index.mapper.MapperService; | ||
|
|
||
| /** PerFieldMappingPostingFormatCodec. {@link org.opensearch.index.codec.PerFieldMappingPostingFormatCodec} */ | ||
| public class PerFieldMappingPostingFormatCodec extends Lucene95CustomCodec { | ||
|
|
||
| /** | ||
| * Creates a new instance. | ||
| * | ||
| * @param compressionMode The compression mode (ZSTD or ZSTDNODICT). | ||
| * @param mapperService The mapper service. | ||
| */ | ||
| public PerFieldMappingPostingFormatCodec(Lucene95CustomCodec.Mode compressionMode, MapperService mapperService) { | ||
| super(compressionMode); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
Oops, something went wrong.