From 3b5b41d3f8bb68f1aa2bf1a696ef41b3391cda70 Mon Sep 17 00:00:00 2001 From: Alexander Lin Date: Sun, 28 Aug 2016 20:08:02 -0700 Subject: [PATCH 1/2] Exposing lucene 6.x minhash filter. --- .../analysis/MinHashTokenFilterFactory.java | 57 +++++++++++++++ .../indices/analysis/AnalysisModule.java | 2 + .../analysis/MinHashFilterFactoryTests.java | 70 +++++++++++++++++++ docs/reference/analysis/tokenfilters.asciidoc | 4 +- .../tokenfilters/minhash-tokenfilter.asciidoc | 22 ++++++ .../AnalysisFactoryTestCase.java | 15 ++-- 6 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/index/analysis/MinHashTokenFilterFactory.java create mode 100644 core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java create mode 100644 docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc diff --git a/core/src/main/java/org/elasticsearch/index/analysis/MinHashTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/MinHashTokenFilterFactory.java new file mode 100644 index 0000000000000..19213dffe2ab9 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/analysis/MinHashTokenFilterFactory.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.minhash.MinHashFilterFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; + +import java.util.HashMap; +import java.util.Map; + +/** + * TokenFilterFactoryAdapter for {@link MinHashFilterFactory} + * + */ +public class MinHashTokenFilterFactory extends AbstractTokenFilterFactory { + + private final MinHashFilterFactory minHashFilterFactory; + + public MinHashTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); + minHashFilterFactory = new MinHashFilterFactory(convertSettings(settings)); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return minHashFilterFactory.create(tokenStream); + } + + private Map convertSettings(Settings settings) { + Map settingMap = new HashMap<>(); + settingMap.put("hashCount", settings.get("hash_count")); + settingMap.put("bucketCount", settings.get("bucket_count")); + settingMap.put("hashSetSize", settings.get("hash_set_size")); + settingMap.put("withRotation", settings.get("with_rotation")); + return settingMap; + } +} diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java index 52647c8b8ff56..5dd0203d61720 100644 --- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java +++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java @@ -90,6 +90,7 @@ import org.elasticsearch.index.analysis.LowerCaseTokenFilterFactory; import org.elasticsearch.index.analysis.LowerCaseTokenizerFactory; import org.elasticsearch.index.analysis.MappingCharFilterFactory; +import org.elasticsearch.index.analysis.MinHashTokenFilterFactory; import org.elasticsearch.index.analysis.NGramTokenFilterFactory; import org.elasticsearch.index.analysis.NGramTokenizerFactory; import org.elasticsearch.index.analysis.NorwegianAnalyzerProvider; @@ -214,6 +215,7 @@ private NamedRegistry> setupTokenFilters(Li tokenFilters.register("edgeNGram", EdgeNGramTokenFilterFactory::new); tokenFilters.register("edge_ngram", EdgeNGramTokenFilterFactory::new); tokenFilters.register("shingle", ShingleTokenFilterFactory::new); + tokenFilters.register("min_hash", MinHashTokenFilterFactory::new); tokenFilters.register("unique", UniqueTokenFilterFactory::new); tokenFilters.register("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new)); tokenFilters.register("trim", TrimTokenFilterFactory::new); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java new file mode 100644 index 0000000000000..60f01cac70051 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTokenStreamTestCase; + +import java.io.IOException; +import java.io.StringReader; + +public class MinHashFilterFactoryTests extends ESTokenStreamTestCase { + public void testDefault() throws IOException { + int default_hash_count = 1; + int default_bucket_size = 512; + int default_hash_set_size = 1; + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); + TokenFilterFactory tokenFilter = analysisService.tokenFilter("min_hash"); + String source = "the quick brown fox"; + Tokenizer tokenizer = new WhitespaceTokenizer(); + tokenizer.setReader(new StringReader(source)); + + // with_rotation is true by default, and hash_set_size is 1, so even though the source doesn't + // have enough tokens to fill all the buckets, we still expect 512 tokens. + assertStreamHasNumberOfTokens(tokenFilter.create(tokenizer), + default_hash_count * default_bucket_size * default_hash_set_size); + } + + public void testSettings() throws IOException { + Settings settings = Settings.builder() + .put("index.analysis.filter.test_min_hash.type", "min_hash") + .put("index.analysis.filter.test_min_hash.hash_count", "1") + .put("index.analysis.filter.test_min_hash.bucket_count", "2") + .put("index.analysis.filter.test_min_hash.hash_set_size", "1") + .put("index.analysis.filter.test_min_hash.with_rotation", false) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); + TokenFilterFactory tokenFilter = analysisService.tokenFilter("test_min_hash"); + String source = "sushi"; + Tokenizer tokenizer = new WhitespaceTokenizer(); + tokenizer.setReader(new StringReader(source)); + + // despite the fact that bucket_count is 2 and hash_set_size is 1, + // because with_rotation is false, we only expect 1 token here. + assertStreamHasNumberOfTokens(tokenFilter.create(tokenizer), 1); + } +} diff --git a/docs/reference/analysis/tokenfilters.asciidoc b/docs/reference/analysis/tokenfilters.asciidoc index 15a911b8ed925..89cce11a615c1 100644 --- a/docs/reference/analysis/tokenfilters.asciidoc +++ b/docs/reference/analysis/tokenfilters.asciidoc @@ -87,4 +87,6 @@ include::tokenfilters/apostrophe-tokenfilter.asciidoc[] include::tokenfilters/decimal-digit-tokenfilter.asciidoc[] -include::tokenfilters/fingerprint-tokenfilter.asciidoc[] \ No newline at end of file +include::tokenfilters/fingerprint-tokenfilter.asciidoc[] + +include::tokenfilters/minhash-tokenfilter.asciidoc[] \ No newline at end of file diff --git a/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc new file mode 100644 index 0000000000000..8d0625e575a03 --- /dev/null +++ b/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc @@ -0,0 +1,22 @@ +[[analysis-minhash-tokenfilter]] +== Minhash Token Filter + +A token filter of type `minhash` hashes each token of the token stream and divides +the resulting hashes into buckets, keeping the lowest-valued hashes per +bucket. It then returns these hashes as tokens. + +The following are settings that can be set for a `minhash` token filter. + +[cols="<,<", options="header",] +|======================================================================= +|Setting |Description +|`hash_count` |The number of hashes to hash the token stream with. Defaults to `1`. + +|`bucket_count` |The number of buckets to divide the minhashes into. Defaults to `512`. + +|`hash_set_size` |The number of minhashes to keep per bucket. Defaults to `1`. + +|`with_rotation` |Whether or not to fill empty buckets with the value of the first non-empty +bucket to its circular right. Only takes effect if hash_set_size is equal to one. +Defaults to `true` if bucket_count is greater than one, else `false`. +|======================================================================= diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index 1b4fcd2922a88..c752563b80420 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -53,6 +53,7 @@ import org.elasticsearch.index.analysis.LowerCaseTokenFilterFactory; import org.elasticsearch.index.analysis.LowerCaseTokenizerFactory; import org.elasticsearch.index.analysis.MappingCharFilterFactory; +import org.elasticsearch.index.analysis.MinHashTokenFilterFactory; import org.elasticsearch.index.analysis.MultiTermAwareComponent; import org.elasticsearch.index.analysis.NGramTokenFilterFactory; import org.elasticsearch.index.analysis.NGramTokenizerFactory; @@ -93,7 +94,7 @@ import java.util.Set; import java.util.TreeSet; -/** +/** * Alerts us if new analyzers are added to lucene, so we don't miss them. *

* If we don't want to expose one for a specific reason, just map it to Void. @@ -115,11 +116,11 @@ public class AnalysisFactoryTestCase extends ESTestCase { .put("thai", ThaiTokenizerFactory.class) .put("uax29urlemail", UAX29URLEmailTokenizerFactory.class) .put("whitespace", WhitespaceTokenizerFactory.class) - + // this one "seems to mess up offsets". probably shouldn't be a tokenizer... .put("wikipedia", Void.class) .immutableMap(); - + static final Map> KNOWN_TOKENFILTERS = new MapBuilder>() // exposed in ES .put("apostrophe", ApostropheFilterFactory.class) @@ -184,6 +185,7 @@ public class AnalysisFactoryTestCase extends ESTestCase { .put("scandinaviannormalization", ScandinavianNormalizationFilterFactory.class) .put("serbiannormalization", SerbianNormalizationFilterFactory.class) .put("shingle", ShingleTokenFilterFactory.class) + .put("minhash", MinHashTokenFilterFactory.class) .put("snowballporter", SnowballTokenFilterFactory.class) .put("soraninormalization", SoraniNormalizationFilterFactory.class) .put("soranistem", StemmerTokenFilterFactory.class) @@ -199,7 +201,7 @@ public class AnalysisFactoryTestCase extends ESTestCase { .put("type", KeepTypesFilterFactory.class) .put("uppercase", UpperCaseTokenFilterFactory.class) .put("worddelimiter", WordDelimiterTokenFilterFactory.class) - + // TODO: these tokenfilters are not yet exposed: useful? // suggest stop @@ -228,16 +230,15 @@ public class AnalysisFactoryTestCase extends ESTestCase { .put("fingerprint", Void.class) // for tee-sinks .put("daterecognizer", Void.class) - .put("minhash", Void.class) .immutableMap(); - + static final Map> KNOWN_CHARFILTERS = new MapBuilder>() // exposed in ES .put("htmlstrip", HtmlStripCharFilterFactory.class) .put("mapping", MappingCharFilterFactory.class) .put("patternreplace", PatternReplaceCharFilterFactory.class) - + // TODO: these charfilters are not yet exposed: useful? // handling of zwnj for persian .put("persian", Void.class) From 355ef6619ae8a639a81e8ab5b33ad9d29b970bed Mon Sep 17 00:00:00 2001 From: Alexander Lin Date: Mon, 29 Aug 2016 19:54:43 -0700 Subject: [PATCH 2/2] fixing documentation --- .../analysis/tokenfilters/minhash-tokenfilter.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc index 8d0625e575a03..0660b0b0e906b 100644 --- a/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc @@ -1,11 +1,11 @@ [[analysis-minhash-tokenfilter]] == Minhash Token Filter -A token filter of type `minhash` hashes each token of the token stream and divides +A token filter of type `min_hash` hashes each token of the token stream and divides the resulting hashes into buckets, keeping the lowest-valued hashes per bucket. It then returns these hashes as tokens. -The following are settings that can be set for a `minhash` token filter. +The following are settings that can be set for a `min_hash` token filter. [cols="<,<", options="header",] |=======================================================================