From c03dc21456403fbc356e3920e2d14cafdfdbadd8 Mon Sep 17 00:00:00 2001 From: anton kolhun Date: Fri, 5 Aug 2022 13:31:14 +0300 Subject: [PATCH] added defaultIndex processing for userInput parsing --- .../prelude/query/parser/AbstractParser.java | 6 ++-- .../prelude/query/parser/AdvancedParser.java | 2 +- .../yahoo/prelude/query/parser/AllParser.java | 8 ++--- .../yahoo/prelude/query/parser/AnyParser.java | 4 +-- .../prelude/query/parser/PhraseParser.java | 2 +- .../prelude/query/parser/SimpleParser.java | 12 ++++--- .../query/parser/StructuredParser.java | 8 +++++ .../prelude/query/parser/TokenizeParser.java | 2 +- .../yahoo/prelude/query/parser/WebParser.java | 4 +-- .../yahoo/search/yql/UserInputTestCase.java | 33 ++++++++++++++++++- 10 files changed, 62 insertions(+), 19 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 586d1d32d574..55001ae59157 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -143,7 +143,7 @@ private Item parse(String queryToParse, String filterToParse, Language parsingLa } setState(parsingLanguage, indexFacts); - Item root = parseItems(); + Item root = parseItems(defaultIndexName); if (filterToParse != null) { AnyParser filterParser = new AnyParser(environment); if (root == null) { @@ -222,8 +222,8 @@ private boolean is(Token.Kind kind, Token tokenOrNull) { if (tokenOrNull == null) return false; return kind.equals(tokenOrNull.kind); } - - protected abstract Item parseItems(); + + protected abstract Item parseItems(String defaultIndexName); /** * Assigns the default index to query terms having no default index. The diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index 690fc67af7e8..3358075d670f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -20,7 +20,7 @@ public AdvancedParser(ParserEnvironment environment) { super(environment); } - protected Item parseItems() { + protected Item parseItems(String defaultIndexName) { return advancedItems(true); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index 545bb8e777fe..09caa72ca59c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -41,16 +41,16 @@ public AllParser(ParserEnvironment environment, boolean weakAnd) { } @Override - protected Item parseItems() { + protected Item parseItems(String defaultIndexName) { int position = tokens.getPosition(); try { - return parseItemsBody(); + return parseItemsBody(defaultIndexName); } finally { tokens.setPosition(position); } } - protected Item parseItemsBody() { + protected Item parseItemsBody(String defaultIndexName) { // Algorithm: Collect positive, negative, and and'ed items, then combine. CompositeItem and = null; NotItem not = null; // Store negatives here as we go @@ -65,7 +65,7 @@ protected Item parseItemsBody() { current = positiveItem(); if (current == null) - current = indexableItem(); + current = indexableItem(defaultIndexName); if (current == null) current = compositeItem(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java index e22043c6b8f3..f4ff769ad053 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java @@ -31,8 +31,8 @@ public AnyParser(ParserEnvironment environment) { super(environment); } - protected Item parseItems() { - return anyItems(true); + protected Item parseItems(String defaultIndexName) { + return anyItems(true, defaultIndexName); } Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java index 75edf9fbf5c0..72eb56dd0fbb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java @@ -16,7 +16,7 @@ public PhraseParser(ParserEnvironment environment) { super(environment); } - protected Item parseItems() { + protected Item parseItems(String defaultIndex) { return forcedPhrase(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 27bce6bd0275..fafbf55a5229 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -33,12 +33,12 @@ protected Item handleComposite(boolean topLevel) { * If there's a explicit composite and some other terms, * a rank terms combines them */ - protected Item anyItems(boolean topLevel) { + protected Item anyItems(boolean topLevel, String defaultIndexName) { int position = tokens.getPosition(); Item item = null; try { - item = anyItemsBody(topLevel); + item = anyItemsBody(topLevel, defaultIndexName); return item; } finally { if (item == null) { @@ -47,7 +47,11 @@ protected Item anyItems(boolean topLevel) { } } - private Item anyItemsBody(boolean topLevel) { + protected Item anyItems(boolean topLevel) { + return anyItems(topLevel, null); + } + + private Item anyItemsBody(boolean topLevel, String defaultIndexName) { Item topLevelItem = null; NotItem not = null; Item item = null; @@ -88,7 +92,7 @@ private Item anyItemsBody(boolean topLevel) { } if (item == null) { - item = indexableItem(); + item = indexableItem(defaultIndexName); if (item != null) { if (topLevelItem == null) { topLevelItem = item; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index f993c7a9e024..c668cf66447e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import static com.yahoo.prelude.query.parser.Token.Kind.*; @@ -52,11 +53,18 @@ protected void setSubmodeFromIndex(String indexName, IndexFacts.Session indexFac } protected Item indexableItem() { + return indexableItem(null); + } + + protected Item indexableItem(String defaultIndexName) { int position = tokens.getPosition(); Item item = null; try { String indexName = indexPrefix(); + if (Objects.isNull(indexName)) { + indexName = defaultIndexName; + } setSubmodeFromIndex(indexName, indexFacts); item = number(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java index dbbc321d0575..eefbe5fa0d06 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java @@ -22,7 +22,7 @@ public TokenizeParser(ParserEnvironment environment) { } @Override - protected Item parseItems() { + protected Item parseItems(String defaultIndex) { WeakAndItem weakAnd = new WeakAndItem(); Token token; while (null != (token = tokens.next())) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java index d7c7dec4798a..40497d94a6d8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java @@ -28,7 +28,7 @@ public WebParser(ParserEnvironment environment) { } @Override - protected Item parseItemsBody() { + protected Item parseItemsBody(String defaultIndexName) { // Algorithm: Collect positive, negative, and'ed and or'ed elements, then combine. CompositeItem and = null; OrItem or = null; @@ -45,7 +45,7 @@ protected Item parseItemsBody() { current = positiveItem(); if (current == null) - current = indexableItem(); + current = indexableItem(defaultIndexName); if (current != null) { if (and != null && (current instanceof WordItem) && "OR".equals(((WordItem)current).getRawWord())) { diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index 8fe451dd0950..1e3b52c23af6 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -3,7 +3,10 @@ import static org.junit.jupiter.api.Assertions.*; -import com.yahoo.search.query.QueryTree; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; import org.apache.http.client.utils.URIBuilder; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -367,4 +370,32 @@ void testUserInputWithEmptyRangeStart() { assertEquals("select * from sources * where text_field contains \"boom\"", query.yqlRepresentation()); } + @Test + void testUserInputWithPhraseSegmentingIndex() { + execution = new Execution(searchChain, Execution.Context.createContextStub(createIndexFacts(true))); + URIBuilder builder = searchUri(); + builder.setParameter("wql", "foo&bar"); + builder.setParameter("yql", "select * from sources * where ([{\"defaultIndex\": \"text_field\",\"grammar\": \"any\"}]userInput(@wql))"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where text_field contains phrase(\"foo\", \"bar\")", query.yqlRepresentation()); + } + + @Test + void testUserInputWithNonPhraseSegmentingIndex() { + execution = new Execution(searchChain, Execution.Context.createContextStub(createIndexFacts(false))); + URIBuilder builder = searchUri(); + builder.setParameter("wql", "foo&bar"); + builder.setParameter("yql", "select * from sources * where ([{\"defaultIndex\": \"text_field\",\"grammar\": \"any\"}]userInput(@wql))"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where (text_field contains \"foo\" AND text_field contains \"bar\")", query.yqlRepresentation()); + } + + private IndexFacts createIndexFacts(boolean phraseSegmenting) { + SearchDefinition sd = new SearchDefinition("sources"); + Index test = new Index("text_field"); + test.setPhraseSegmenting(phraseSegmenting); + sd.addIndex(test); + return new IndexFacts(new IndexModel(sd)); + } + }