Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for wildcard_query function #156

Merged
merged 35 commits into from
Nov 25, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
a96a740
Implemented wildcard_query and added tests in core
Nov 2, 2022
885f04c
Implemented and added tests for sql
Nov 2, 2022
4d0dadb
Implemented and added tests for ppl
Nov 3, 2022
9c25e66
Implemented and added tests for lucene
Nov 3, 2022
efe0ee5
Fixed test for like expression
Nov 3, 2022
bedf662
Added parameters to wildcard_query
Nov 4, 2022
afbeb44
Added integration tests for ppl and sql
Nov 4, 2022
e25d4da
Added docs for doctests
Nov 4, 2022
84fcf88
Pulled from 2.x
Nov 7, 2022
4708ae2
Fixed issues introduced during merging
Nov 7, 2022
e05e5c5
Addressed PR comment
Nov 7, 2022
988b40b
Added annotation that was deleted from merging
Nov 8, 2022
5c225ce
Fixed merge conflict issues
Nov 8, 2022
0b9752c
Addressed some PR comments and handled escaping wildcards
Nov 9, 2022
a2ca906
Added tests for wildcard conversion and created data for testing
Nov 11, 2022
68c97d6
Added javadoc
Nov 14, 2022
a873b27
Changed index name
Nov 14, 2022
518c09f
Temporarily changed jackson_version to run GH actions
Nov 14, 2022
1101ca3
Added comparison test for wildcard conversion
Nov 14, 2022
fc8883c
Removed PPL implementation of wildcard_query
Nov 14, 2022
ae46f96
Reverted ppl docs change
Nov 14, 2022
a9f5be7
Made namedArgument a static function
Nov 14, 2022
bba63ce
Removed extra space
Nov 14, 2022
8b7b442
Merge branch 'integ-add-wildcardquery' of github.com:Bit-Quill/opense…
Nov 15, 2022
4af9eef
Fixed LIKE query
Nov 17, 2022
57aac8e
Fixed LIKE tests and added more tests
Nov 18, 2022
0a4af9d
Addressed PR comments
Nov 18, 2022
288e29c
Implemented converting text field to keyword. Still needs testing
Nov 21, 2022
1fb4973
Added test cases for LIKE in sql and ppl
Nov 22, 2022
34ffe13
Addressed PR comments regarding docs
Nov 24, 2022
eed3294
Fixed backslashes in docs
Nov 24, 2022
86fa737
Added missed backticks in docs
Nov 24, 2022
b54a934
Moved escaping wildcard test to common/utils
Nov 25, 2022
fede123
Fixed merge conflicts
Nov 25, 2022
9da6140
Fixed checkstyle error
Nov 25, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,48 @@ public static String format(final String format, Object... args) {
return String.format(Locale.ROOT, format, args);
}

/**
* Converts sql wildcard character % and _ to * and ?.
* @param text string to be converted
* @return converted string
*/
public static String convertSqlWildcardToLucene(String text) {
final char DEFAULT_ESCAPE = '\\';
StringBuilder convertedString = new StringBuilder(text.length());
boolean escaped = false;

for (char currentChar : text.toCharArray()) {
switch (currentChar) {
case DEFAULT_ESCAPE:
escaped = true;
convertedString.append(currentChar);
break;
case '%':
if (escaped) {
convertedString.deleteCharAt(convertedString.length() - 1);
convertedString.append("%");
} else {
convertedString.append("*");
}
escaped = false;
break;
case '_':
if (escaped) {
convertedString.deleteCharAt(convertedString.length() - 1);
convertedString.append("_");
} else {
convertedString.append('?');
}
escaped = false;
break;
default:
convertedString.append(currentChar);
escaped = false;
}
}
return convertedString.toString();
}

private static boolean isQuoted(String text, String mark) {
return !Strings.isNullOrEmpty(text) && text.startsWith(mark) && text.endsWith(mark);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,19 @@ public void test_wildcard_query_asterisk_function() throws IOException {
verifyDataRows(result2, rows(expected));
}

@Test
public void test_LIKE() throws IOException {
String expected = "test wildcard";

String query1 = "SELECT Body, Body LIKE 't*' FROM " + TEST_INDEX_WILDCARD + " LIMIT 1";
JSONObject result1 = executeJdbcRequest(query1);
verifyDataRows(result1, rows(expected));

query1 = "SELECT Body FROM " + TEST_INDEX_WILDCARD + " WHERE Body LIKE 't*' LIMIT 1";
result1 = executeJdbcRequest(query1);
verifyDataRows(result1, rows(expected));
}

@Test
public void test_wildcard_query_question_mark_function() throws IOException {
String expected = "test wildcard";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.opensearch.sql.expression.FunctionExpression;
import org.opensearch.sql.expression.function.BuiltinFunctionName;
import org.opensearch.sql.expression.function.FunctionName;
import org.opensearch.sql.opensearch.storage.script.filter.lucene.LikeQuery;
import org.opensearch.sql.opensearch.storage.script.filter.lucene.LuceneQuery;
import org.opensearch.sql.opensearch.storage.script.filter.lucene.RangeQuery;
import org.opensearch.sql.opensearch.storage.script.filter.lucene.RangeQuery.Comparison;
Expand Down Expand Up @@ -57,7 +58,7 @@ public class FilterQueryBuilder extends ExpressionNodeVisitor<QueryBuilder, Obje
.put(BuiltinFunctionName.GREATER.getName(), new RangeQuery(Comparison.GT))
.put(BuiltinFunctionName.LTE.getName(), new RangeQuery(Comparison.LTE))
.put(BuiltinFunctionName.GTE.getName(), new RangeQuery(Comparison.GTE))
.put(BuiltinFunctionName.LIKE.getName(), new WildcardQuery())
.put(BuiltinFunctionName.LIKE.getName(), new LikeQuery())
.put(BuiltinFunctionName.MATCH.getName(), new MatchQuery())
.put(BuiltinFunctionName.MATCH_PHRASE.getName(), new MatchPhraseQuery())
.put(BuiltinFunctionName.MATCHPHRASE.getName(), new MatchPhraseQuery())
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.opensearch.storage.script.filter.lucene;

import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.WildcardQueryBuilder;
import org.opensearch.sql.common.utils.StringUtils;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.expression.Expression;
import org.opensearch.sql.expression.FunctionExpression;
import org.opensearch.sql.expression.ReferenceExpression;

public class LikeQuery extends LuceneQuery {
@Override
public QueryBuilder build(FunctionExpression func) {
ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0);
Expression expr = func.getArguments().get(1);
ExprValue literalValue = expr.valueOf();

return createBuilder(ref.toString(), literalValue.stringValue());
}

protected WildcardQueryBuilder createBuilder(String field, String query) {
acarbonetto marked this conversation as resolved.
Show resolved Hide resolved
String matchText = StringUtils.convertSqlWildcardToLucene(query);
return QueryBuilders.wildcardQuery(field, matchText);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.WildcardQueryBuilder;
import org.opensearch.sql.common.utils.StringUtils;

/**
* Lucene query that builds wildcard query.
Expand All @@ -21,57 +22,14 @@ public WildcardQuery() {
super(FunctionParameterRepository.WildcardQueryBuildActions);
}

private static final char DEFAULT_ESCAPE = '\\';

/**
* Converts sql wildcard character % and _ to * and ?.
* @param text string to be converted
* @return converted string
*/
public String convertSqlWildcardToLucene(String text) {
StringBuilder convertedString = new StringBuilder(text.length());
boolean escaped = false;

for (char currentChar : text.toCharArray()) {
switch (currentChar) {
case DEFAULT_ESCAPE:
escaped = true;
convertedString.append(currentChar);
break;
case '%':
if (escaped) {
convertedString.deleteCharAt(convertedString.length() - 1);
convertedString.append("%");
} else {
convertedString.append("*");
}
escaped = false;
break;
case '_':
if (escaped) {
convertedString.deleteCharAt(convertedString.length() - 1);
convertedString.append("_");
} else {
convertedString.append('?');
}
escaped = false;
break;
default:
convertedString.append(currentChar);
escaped = false;
}
}
return convertedString.toString();
}

@Override
protected String getQueryName() {
return WildcardQueryBuilder.NAME;
}

@Override
protected WildcardQueryBuilder createBuilder(String field, String query) {
String matchText = convertSqlWildcardToLucene(query);
String matchText = StringUtils.convertSqlWildcardToLucene(query);
return QueryBuilders.wildcardQuery(field, matchText);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.opensearch.sql.common.antlr.SyntaxCheckException;
import org.opensearch.sql.common.utils.StringUtils;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.exception.SemanticCheckException;
Expand Down Expand Up @@ -77,19 +78,19 @@ public void test_SemanticCheckException_when_invalid_parameter() {

@Test
public void test_escaping_sql_wildcards() {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A better home for this test is in the common/src/test/java/org.opensearch.sql.common.utils -- in same module where the StringUtils.convertSqlWildcardToLucene is defined.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. Addressed in b54a934

assertEquals("%", wildcardQueryQuery.convertSqlWildcardToLucene("\\%"));
assertEquals("\\*", wildcardQueryQuery.convertSqlWildcardToLucene("\\*"));
assertEquals("_", wildcardQueryQuery.convertSqlWildcardToLucene("\\_"));
assertEquals("\\?", wildcardQueryQuery.convertSqlWildcardToLucene("\\?"));
assertEquals("%*", wildcardQueryQuery.convertSqlWildcardToLucene("\\%%"));
assertEquals("*%", wildcardQueryQuery.convertSqlWildcardToLucene("%\\%"));
assertEquals("%*%", wildcardQueryQuery.convertSqlWildcardToLucene("\\%%\\%"));
assertEquals("*%*", wildcardQueryQuery.convertSqlWildcardToLucene("%\\%%"));
assertEquals("_?", wildcardQueryQuery.convertSqlWildcardToLucene("\\__"));
assertEquals("?_", wildcardQueryQuery.convertSqlWildcardToLucene("_\\_"));
assertEquals("_?_", wildcardQueryQuery.convertSqlWildcardToLucene("\\__\\_"));
assertEquals("?_?", wildcardQueryQuery.convertSqlWildcardToLucene("_\\__"));
assertEquals("%\\*_\\?", wildcardQueryQuery.convertSqlWildcardToLucene("\\%\\*\\_\\?"));
assertEquals("%", StringUtils.convertSqlWildcardToLucene("\\%"));
assertEquals("\\*", StringUtils.convertSqlWildcardToLucene("\\*"));
assertEquals("_", StringUtils.convertSqlWildcardToLucene("\\_"));
assertEquals("\\?", StringUtils.convertSqlWildcardToLucene("\\?"));
assertEquals("%*", StringUtils.convertSqlWildcardToLucene("\\%%"));
assertEquals("*%", StringUtils.convertSqlWildcardToLucene("%\\%"));
assertEquals("%*%", StringUtils.convertSqlWildcardToLucene("\\%%\\%"));
assertEquals("*%*", StringUtils.convertSqlWildcardToLucene("%\\%%"));
assertEquals("_?", StringUtils.convertSqlWildcardToLucene("\\__"));
assertEquals("?_", StringUtils.convertSqlWildcardToLucene("_\\_"));
assertEquals("_?_", StringUtils.convertSqlWildcardToLucene("\\__\\_"));
assertEquals("?_?", StringUtils.convertSqlWildcardToLucene("_\\__"));
assertEquals("%\\*_\\?", StringUtils.convertSqlWildcardToLucene("\\%\\*\\_\\?"));
}

private class WildcardQueryExpression extends FunctionExpression {
Expand Down