From 461fc6ad599e5346edd2c44cd322df98215a0477 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Fri, 22 Mar 2024 13:03:05 -0400 Subject: [PATCH 01/12] Starting point for zulia testing framework --- gradle/libs.versions.toml | 2 +- settings.gradle | 3 +- zulia-testing/build.gradle.kts | 14 +++ .../io/zulia/testing/ConnectionConfig.java | 35 +++++++ .../java/io/zulia/testing/IndexConfig.java | 32 ++++++ .../java/io/zulia/testing/QueryConfig.java | 34 +++++++ .../java/io/zulia/testing/TestConfig.java | 22 +++++ .../java/io/zulia/testing/TestRunner.java | 98 +++++++++++++++++++ .../io/zulia/testing/ZuliaTestConfig.java | 52 ++++++++++ zulia-testing/src/main/resources/testing.yaml | 40 ++++++++ .../util/exception/ThrowingSupplier.java | 19 ++++ 11 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 zulia-testing/build.gradle.kts create mode 100644 zulia-testing/src/main/java/io/zulia/testing/ConnectionConfig.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/IndexConfig.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/TestConfig.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/TestRunner.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/ZuliaTestConfig.java create mode 100644 zulia-testing/src/main/resources/testing.yaml create mode 100644 zulia-util/src/main/java/io/zulia/util/exception/ThrowingSupplier.java diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 74f3a061..5806267d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -22,7 +22,7 @@ guava = { module = "com.google.guava:guava", version = "32.1.3-jre" } grpc-netty-shaded = { module = "io.grpc:grpc-netty-shaded", version.ref = "grpc" } grpc-protobuf = { module = "io.grpc:grpc-protobuf", version.ref = "grpc" } grpc-stub = { module = "io.grpc:grpc-stub", version.ref = "grpc" } -#graalvm-js = {module = "org.graalvm.js:js", version = "23.0.3"} +graalvm-js = { module = "org.graalvm.js:js", version = "23.0.3" } gson = { module = "com.google.code.gson:gson", version = "2.10.1" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version = "2.16.1" } jakarta-validation = { module = "jakarta.validation:jakarta.validation-api" } diff --git a/settings.gradle b/settings.gradle index 2b4dd46b..44c1c14e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,5 @@ rootProject.name = "ZuliaSearch" -include 'zulia-analyzer', 'zulia-common', 'zulia-client', 'zulia-data', 'zulia-query-parser', 'zulia-server', 'zulia-util' +include 'zulia-analyzer', 'zulia-common', 'zulia-client', 'zulia-data', 'zulia-query-parser', 'zulia-server', 'zulia-testing', 'zulia-util' + diff --git a/zulia-testing/build.gradle.kts b/zulia-testing/build.gradle.kts new file mode 100644 index 00000000..5d64f63e --- /dev/null +++ b/zulia-testing/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `java-library` +} + +description = "Zulia Testing" + + +dependencies { + api(project(":zulia-util")) + api(project(":zulia-client")) + implementation(libs.snake.yaml) + implementation(libs.graalvm.js) +} + diff --git a/zulia-testing/src/main/java/io/zulia/testing/ConnectionConfig.java b/zulia-testing/src/main/java/io/zulia/testing/ConnectionConfig.java new file mode 100644 index 00000000..1ab93c30 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/ConnectionConfig.java @@ -0,0 +1,35 @@ +package io.zulia.testing; + +public class ConnectionConfig { + + private String serverAddress; + + private int port; + + public ConnectionConfig() { + + } + + public String getServerAddress() { + return serverAddress; + } + + public void setServerAddress(String serverAddress) { + this.serverAddress = serverAddress; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + @Override + public String toString() { + return "ConnectionConfig{" + "serverAddress='" + serverAddress + '\'' + ", port=" + port + '}'; + } +} + + diff --git a/zulia-testing/src/main/java/io/zulia/testing/IndexConfig.java b/zulia-testing/src/main/java/io/zulia/testing/IndexConfig.java new file mode 100644 index 00000000..11d07785 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/IndexConfig.java @@ -0,0 +1,32 @@ +package io.zulia.testing; + +public final class IndexConfig { + + private String indexName; + + private String connection; + + public IndexConfig() { + } + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getConnection() { + return connection; + } + + public void setConnection(String connection) { + this.connection = connection; + } + + @Override + public String toString() { + return "IndexConfig{" + "indexName='" + indexName + '\'' + ", connection='" + connection + '\'' + '}'; + } +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java b/zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java new file mode 100644 index 00000000..dc579bfe --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java @@ -0,0 +1,34 @@ +package io.zulia.testing; + +public final class QueryConfig { + private String index; + private String query; + + private int amount; + + public QueryConfig() { + + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + @Override + public String toString() { + return "QueryConfig[" + "index=" + index + ", " + "query=" + query + ']'; + } + +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/TestConfig.java b/zulia-testing/src/main/java/io/zulia/testing/TestConfig.java new file mode 100644 index 00000000..e8d4ef2c --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/TestConfig.java @@ -0,0 +1,22 @@ +package io.zulia.testing; + +public final class TestConfig { + private String expr; + + public TestConfig() { + + } + + public String getExpr() { + return expr; + } + + public void setExpr(String expr) { + this.expr = expr; + } + + @Override + public String toString() { + return "TestConfig{" + "expr='" + expr + '\'' + '}'; + } +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java b/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java new file mode 100644 index 00000000..fe880535 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java @@ -0,0 +1,98 @@ +package io.zulia.testing; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import io.zulia.client.command.builder.FilterQuery; +import io.zulia.client.command.builder.Search; +import io.zulia.client.config.ZuliaPoolConfig; +import io.zulia.client.pool.ZuliaWorkPool; +import io.zulia.client.result.SearchResult; +import io.zulia.util.exception.ThrowingSupplier; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.HostAccess; +import org.graalvm.polyglot.Value; +import org.yaml.snakeyaml.Yaml; + +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +public class TestRunner { + + public static class Result { + + public Result(long count) { + this.count = count; + } + + public long count; + } + + public static void main(String[] args) throws Exception { + Yaml yaml = new Yaml(); + + ZuliaTestConfig dataQualityConfig; + try (InputStream inputStream = TestRunner.class.getResource("/testing.yaml").openStream()) { + dataQualityConfig = yaml.loadAs(inputStream, ZuliaTestConfig.class); + } + + Map> connectionToConnectionConfig = new HashMap<>(); + for (Map.Entry indexConfigEntry : dataQualityConfig.getConnections().entrySet()) { + ConnectionConfig connectionConfig = indexConfigEntry.getValue(); + + ThrowingSupplier throwingSupplier = () -> { + ZuliaPoolConfig zuliaPoolConfig = new ZuliaPoolConfig().addNode(connectionConfig.getServerAddress(), connectionConfig.getPort()); + return new ZuliaWorkPool(zuliaPoolConfig); + }; + + connectionToConnectionConfig.put(indexConfigEntry.getKey(), Suppliers.memoize(throwingSupplier)); + } + + for (Map.Entry indexConfigEntry : dataQualityConfig.getIndexes().entrySet()) { + IndexConfig indexConfig = indexConfigEntry.getValue(); + Supplier zuliaWorkPoolSupplier = connectionToConnectionConfig.get(indexConfig.getConnection()); + if (zuliaWorkPoolSupplier == null) { + throw new IllegalArgumentException( + "Failed to find connection config <" + indexConfig.getConnection() + "> for query <" + indexConfigEntry.getKey() + ">"); + } + } + + Map countMap = new HashMap<>(); + for (Map.Entry queryConfigEntry : dataQualityConfig.getQueries().entrySet()) { + String queryId = queryConfigEntry.getKey(); + QueryConfig queryConfig = queryConfigEntry.getValue(); + IndexConfig indexConfig = dataQualityConfig.getIndexes().get(queryConfig.getIndex()); + Supplier zuliaWorkPoolSupplier = connectionToConnectionConfig.get(indexConfig.getConnection()); + ZuliaWorkPool zuliaWorkPool = zuliaWorkPoolSupplier.get(); + + Search s = new Search(indexConfig.getIndexName()); + s.addQuery(new FilterQuery(queryConfig.getQuery())); + + SearchResult searchResult = zuliaWorkPool.search(s); + countMap.put(queryId, searchResult.getTotalHits()); + + } + + try (Context context = Context.newBuilder("js").allowHostAccess(HostAccess.ALL).build()) { + Value jsValue = context.getBindings("js"); + for (Map.Entry countToEntry : countMap.entrySet()) { + jsValue.putMember(countToEntry.getKey(), new Result(countToEntry.getValue())); + } + + for (Map.Entry testConfigEntry : dataQualityConfig.getTests().entrySet()) { + TestConfig testConfig = testConfigEntry.getValue(); + + boolean testResult = context.eval("js", testConfig.getExpr()).asBoolean(); + if (!testResult) { + System.out.println(testConfigEntry.getKey() + " has failed"); + } + else { + System.out.println(testConfigEntry.getKey() + " has passed"); + } + + } + + } + + } +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestConfig.java b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestConfig.java new file mode 100644 index 00000000..dcaafd4b --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestConfig.java @@ -0,0 +1,52 @@ +package io.zulia.testing; + +import java.util.Map; + +public class ZuliaTestConfig { + + private Map connections; + private Map indexes; + private Map queries; + private Map tests; + + public ZuliaTestConfig() { + + } + + public Map getConnections() { + return connections; + } + + public void setConnections(Map connections) { + this.connections = connections; + } + + public Map getIndexes() { + return indexes; + } + + public void setIndexes(Map indexes) { + this.indexes = indexes; + } + + public Map getQueries() { + return queries; + } + + public void setQueries(Map queries) { + this.queries = queries; + } + + public Map getTests() { + return tests; + } + + public void setTests(Map tests) { + this.tests = tests; + } + + @Override + public String toString() { + return "DataQualityConfig{" + "connections=" + connections + ", indexes=" + indexes + ", queries=" + queries + ", tests=" + tests + '}'; + } +} diff --git a/zulia-testing/src/main/resources/testing.yaml b/zulia-testing/src/main/resources/testing.yaml new file mode 100644 index 00000000..45b46a24 --- /dev/null +++ b/zulia-testing/src/main/resources/testing.yaml @@ -0,0 +1,40 @@ +connections: + testConnection: + serverAddress: 192.168.1.30 + port: 32191 +indexes: + publications: + indexName: publications_2024 + connection: testConnection +queries: + pubsStar: + index: publications + query: "*:*" + pubsWithoutTitle: + index: publications + query: "-title:*" + pubsTest: + index: publications + query: "-title:*" + amount: 1 + facet: + - field: pubYear + topN: 10 + - field: journalName + topN: 10 + + +tests: + pubsCount: + expr: pubsStar.count > 30000000 + pubsWithTitleCount: + expr: pubsWithoutTitle.count < pubsStar.count * 0.01 + test1: + expr: pubsTest.count > 1 + test2: + expr: pubsTest.doc[0]["title"].length == 0 + test3: + expr: pubsTest.facet["pubYear"][1] > 1000000 + + + diff --git a/zulia-util/src/main/java/io/zulia/util/exception/ThrowingSupplier.java b/zulia-util/src/main/java/io/zulia/util/exception/ThrowingSupplier.java new file mode 100644 index 00000000..7ffebc00 --- /dev/null +++ b/zulia-util/src/main/java/io/zulia/util/exception/ThrowingSupplier.java @@ -0,0 +1,19 @@ +package io.zulia.util.exception; + +import com.google.common.base.Supplier; + +public interface ThrowingSupplier extends Supplier { + + T throwingGet() throws Throwable; + + @Override + default T get() { + try { + return throwingGet(); + } + catch (Throwable e) { + throw new RuntimeException(e); + } + } + +} From 53fab22b73c6a4256b19e598379c1115ecd1ca34 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Tue, 2 Apr 2024 13:03:58 -0400 Subject: [PATCH 02/12] add faceting testing and document testing remove warnings from the log --- zulia-testing/build.gradle.kts | 2 + .../java/io/zulia/testing/FacetConfig.java | 31 ++++++ .../java/io/zulia/testing/QueryConfig.java | 23 +++- .../java/io/zulia/testing/TestResult.java | 42 ++++++++ .../java/io/zulia/testing/TestRunner.java | 102 +++++++++++++----- .../testing/js/dto/DocumentProxyObject.java | 69 ++++++++++++ .../testing/js/dto/FacetProxyObject.java | 54 ++++++++++ .../io/zulia/testing/js/dto/FacetValue.java | 7 ++ .../io/zulia/testing/js/dto/QueryResult.java | 18 ++++ zulia-testing/src/main/resources/logback.xml | 16 +++ 10 files changed, 334 insertions(+), 30 deletions(-) create mode 100644 zulia-testing/src/main/java/io/zulia/testing/FacetConfig.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/TestResult.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/js/dto/DocumentProxyObject.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValue.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResult.java create mode 100644 zulia-testing/src/main/resources/logback.xml diff --git a/zulia-testing/build.gradle.kts b/zulia-testing/build.gradle.kts index 5d64f63e..bcdc4752 100644 --- a/zulia-testing/build.gradle.kts +++ b/zulia-testing/build.gradle.kts @@ -10,5 +10,7 @@ dependencies { api(project(":zulia-client")) implementation(libs.snake.yaml) implementation(libs.graalvm.js) + implementation(libs.logback.classic) + implementation(libs.jansi) } diff --git a/zulia-testing/src/main/java/io/zulia/testing/FacetConfig.java b/zulia-testing/src/main/java/io/zulia/testing/FacetConfig.java new file mode 100644 index 00000000..012eed42 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/FacetConfig.java @@ -0,0 +1,31 @@ +package io.zulia.testing; + +public class FacetConfig { + + private String field; + private int topN; + + public FacetConfig() { + } + + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } + + public int getTopN() { + return topN; + } + + public void setTopN(int topN) { + this.topN = topN; + } + + @Override + public String toString() { + return "FacetConfig{" + "field='" + field + '\'' + ", topN=" + topN + '}'; + } +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java b/zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java index dc579bfe..af1ee5f0 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java @@ -1,11 +1,15 @@ package io.zulia.testing; +import java.util.List; + public final class QueryConfig { private String index; private String query; private int amount; + private List facets; + public QueryConfig() { } @@ -26,9 +30,24 @@ public void setQuery(String query) { this.query = query; } + public int getAmount() { + return amount; + } + + public void setAmount(int amount) { + this.amount = amount; + } + + public List getFacets() { + return facets; + } + + public void setFacets(List facets) { + this.facets = facets; + } + @Override public String toString() { - return "QueryConfig[" + "index=" + index + ", " + "query=" + query + ']'; + return "QueryConfig{" + "index='" + index + '\'' + ", query='" + query + '\'' + ", amount=" + amount + ", facets=" + facets + '}'; } - } diff --git a/zulia-testing/src/main/java/io/zulia/testing/TestResult.java b/zulia-testing/src/main/java/io/zulia/testing/TestResult.java new file mode 100644 index 00000000..1ceed420 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/TestResult.java @@ -0,0 +1,42 @@ +package io.zulia.testing; + +public class TestResult { + + private String testId; + + private TestConfig testConfig; + + private boolean passed; + + public TestResult() { + } + + public String getTestId() { + return testId; + } + + public void setTestId(String testId) { + this.testId = testId; + } + + public TestConfig getTestConfig() { + return testConfig; + } + + public void setTestConfig(TestConfig testConfig) { + this.testConfig = testConfig; + } + + public boolean isPassed() { + return passed; + } + + public void setPassed(boolean passed) { + this.passed = passed; + } + + @Override + public String toString() { + return "TestResult{" + "testId='" + testId + '\'' + ", testConfig=" + testConfig + ", passed=" + passed + '}'; + } +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java b/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java index fe880535..f83c88cc 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java +++ b/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java @@ -2,32 +2,34 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import io.zulia.client.command.builder.CountFacet; import io.zulia.client.command.builder.FilterQuery; import io.zulia.client.command.builder.Search; import io.zulia.client.config.ZuliaPoolConfig; import io.zulia.client.pool.ZuliaWorkPool; +import io.zulia.client.result.CompleteResult; import io.zulia.client.result.SearchResult; +import io.zulia.message.ZuliaQuery; +import io.zulia.testing.js.dto.DocumentProxyObject; +import io.zulia.testing.js.dto.FacetProxyObject; +import io.zulia.testing.js.dto.FacetValue; +import io.zulia.testing.js.dto.QueryResult; import io.zulia.util.exception.ThrowingSupplier; +import org.bson.Document; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.HostAccess; import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.proxy.ProxyObject; import org.yaml.snakeyaml.Yaml; import java.io.InputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; public class TestRunner { - public static class Result { - - public Result(long count) { - this.count = count; - } - - public long count; - } - public static void main(String[] args) throws Exception { Yaml yaml = new Yaml(); @@ -36,8 +38,16 @@ public static void main(String[] args) throws Exception { dataQualityConfig = yaml.loadAs(inputStream, ZuliaTestConfig.class); } + List testResults = runTests(dataQualityConfig); + for (TestResult testResult : testResults) { + System.out.println(testResult.getTestId() + " result is " + testResult.isPassed()); + } + + } + + public static List runTests(ZuliaTestConfig zuliaTestConfig) throws Exception { Map> connectionToConnectionConfig = new HashMap<>(); - for (Map.Entry indexConfigEntry : dataQualityConfig.getConnections().entrySet()) { + for (Map.Entry indexConfigEntry : zuliaTestConfig.getConnections().entrySet()) { ConnectionConfig connectionConfig = indexConfigEntry.getValue(); ThrowingSupplier throwingSupplier = () -> { @@ -48,7 +58,7 @@ public static void main(String[] args) throws Exception { connectionToConnectionConfig.put(indexConfigEntry.getKey(), Suppliers.memoize(throwingSupplier)); } - for (Map.Entry indexConfigEntry : dataQualityConfig.getIndexes().entrySet()) { + for (Map.Entry indexConfigEntry : zuliaTestConfig.getIndexes().entrySet()) { IndexConfig indexConfig = indexConfigEntry.getValue(); Supplier zuliaWorkPoolSupplier = connectionToConnectionConfig.get(indexConfig.getConnection()); if (zuliaWorkPoolSupplier == null) { @@ -57,42 +67,78 @@ public static void main(String[] args) throws Exception { } } - Map countMap = new HashMap<>(); - for (Map.Entry queryConfigEntry : dataQualityConfig.getQueries().entrySet()) { + Map resultMap = new HashMap<>(); + for (Map.Entry queryConfigEntry : zuliaTestConfig.getQueries().entrySet()) { String queryId = queryConfigEntry.getKey(); QueryConfig queryConfig = queryConfigEntry.getValue(); - IndexConfig indexConfig = dataQualityConfig.getIndexes().get(queryConfig.getIndex()); + IndexConfig indexConfig = zuliaTestConfig.getIndexes().get(queryConfig.getIndex()); Supplier zuliaWorkPoolSupplier = connectionToConnectionConfig.get(indexConfig.getConnection()); ZuliaWorkPool zuliaWorkPool = zuliaWorkPoolSupplier.get(); Search s = new Search(indexConfig.getIndexName()); s.addQuery(new FilterQuery(queryConfig.getQuery())); + s.setAmount(queryConfig.getAmount()); + + if (queryConfig.getFacets() != null) { + for (FacetConfig facet : queryConfig.getFacets()) { + CountFacet countFacet = new CountFacet(facet.getField()); + if (facet.getTopN() != 0) { + countFacet.setTopN(facet.getTopN()); + } + s.addCountFacet(countFacet); + } + } SearchResult searchResult = zuliaWorkPool.search(s); - countMap.put(queryId, searchResult.getTotalHits()); + QueryResult result = new QueryResult(); + result.count = searchResult.getTotalHits(); + + List docs = new ArrayList<>(); + for (CompleteResult completeResult : searchResult.getCompleteResults()) { + Document document = completeResult.getDocument(); + docs.add(new DocumentProxyObject(document)); + } + Map> facetResults = new HashMap<>(); + + if (queryConfig.getFacets() != null) { + for (FacetConfig facet : queryConfig.getFacets()) { + List facetValues = new ArrayList<>(); + List facetCounts = searchResult.getFacetCounts(facet.getField()); + for (ZuliaQuery.FacetCount facetCount : facetCounts) { + FacetValue fv = new FacetValue(); + fv.label = facetCount.getFacet(); + fv.count = facetCount.getCount(); + facetValues.add(fv); + } + facetResults.put(facet.getField(), facetValues); + } + result.facet = new FacetProxyObject(facetResults); + } + result.doc = docs; + resultMap.put(queryId, result); } - try (Context context = Context.newBuilder("js").allowHostAccess(HostAccess.ALL).build()) { + List testResults = new ArrayList<>(); + try (Context context = Context.newBuilder("js").option("engine.WarnInterpreterOnly", "false").allowHostAccess(HostAccess.ALL).build()) { Value jsValue = context.getBindings("js"); - for (Map.Entry countToEntry : countMap.entrySet()) { - jsValue.putMember(countToEntry.getKey(), new Result(countToEntry.getValue())); + for (Map.Entry countToEntry : resultMap.entrySet()) { + jsValue.putMember(countToEntry.getKey(), countToEntry.getValue()); } - for (Map.Entry testConfigEntry : dataQualityConfig.getTests().entrySet()) { + for (Map.Entry testConfigEntry : zuliaTestConfig.getTests().entrySet()) { TestConfig testConfig = testConfigEntry.getValue(); - boolean testResult = context.eval("js", testConfig.getExpr()).asBoolean(); - if (!testResult) { - System.out.println(testConfigEntry.getKey() + " has failed"); - } - else { - System.out.println(testConfigEntry.getKey() + " has passed"); - } - + TestResult testResult = new TestResult(); + testResult.setTestId(testConfigEntry.getKey()); + testResult.setTestConfig(testConfig); + Value js = context.eval("js", testConfig.getExpr()); + System.out.println(js); + testResult.setPassed(js.asBoolean()); + testResults.add(testResult); } } - + return testResults; } } diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/DocumentProxyObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/DocumentProxyObject.java new file mode 100644 index 00000000..51db7b3b --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/DocumentProxyObject.java @@ -0,0 +1,69 @@ +package io.zulia.testing.js.dto; + +import org.bson.Document; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.proxy.ProxyArray; +import org.graalvm.polyglot.proxy.ProxyObject; + +public class DocumentProxyObject implements ProxyObject { + private final Document document; + + public DocumentProxyObject(Document document) { + this.document = document; + } + + @Override + public void putMember(String key, Value value) { + document.put(key, value.isHostObject() ? value.asHostObject() : value); + } + + @Override + public boolean hasMember(String key) { + return document.containsKey(key); + } + + @Override + public Object getMemberKeys() { + return new ProxyArray() { + private final Object[] keys = document.keySet().toArray(); + + public void set(long index, Value value) { + throw new UnsupportedOperationException(); + } + + public long getSize() { + return keys.length; + } + + public Object get(long index) { + if (index < 0 || index > Integer.MAX_VALUE) { + throw new ArrayIndexOutOfBoundsException(); + } + return keys[(int) index]; + } + }; + } + + @Override + public Object getMember(String key) { + Object v = document.get(key); + if (v instanceof Document d) { + return new DocumentProxyObject(d); + } + else { + return v; + } + } + + @Override + public boolean removeMember(String key) { + if (document.containsKey(key)) { + document.remove(key); + return true; + } + else { + return false; + } + } + +} \ No newline at end of file diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java new file mode 100644 index 00000000..6bb58466 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java @@ -0,0 +1,54 @@ +package io.zulia.testing.js.dto; + +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.proxy.ProxyArray; +import org.graalvm.polyglot.proxy.ProxyObject; + +import java.util.List; +import java.util.Map; + +public class FacetProxyObject implements ProxyObject { + private final Map> keyToFacets; + + public FacetProxyObject(Map> keyToFacets) { + this.keyToFacets = keyToFacets; + } + + @Override + public void putMember(String key, Value value) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasMember(String key) { + return keyToFacets.containsKey(key); + } + + @Override + public Object getMemberKeys() { + return new ProxyArray() { + private final Object[] keys = keyToFacets.keySet().toArray(); + + public void set(long index, Value value) { + throw new UnsupportedOperationException(); + } + + public long getSize() { + return keys.length; + } + + public Object get(long index) { + if (index < 0 || index > Integer.MAX_VALUE) { + throw new ArrayIndexOutOfBoundsException(); + } + return keys[(int) index]; + } + }; + } + + @Override + public List getMember(String key) { + return keyToFacets.get(key); + } + +} \ No newline at end of file diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValue.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValue.java new file mode 100644 index 00000000..7361422c --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValue.java @@ -0,0 +1,7 @@ +package io.zulia.testing.js.dto; + +public class FacetValue { + public String label; + public long count; + +} \ No newline at end of file diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResult.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResult.java new file mode 100644 index 00000000..bbed5e96 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResult.java @@ -0,0 +1,18 @@ +package io.zulia.testing.js.dto; + +import org.graalvm.polyglot.proxy.ProxyObject; + +import java.util.List; + +public class QueryResult { + + public QueryResult() { + + } + + public long count; + + public List doc; + + public FacetProxyObject facet; +} diff --git a/zulia-testing/src/main/resources/logback.xml b/zulia-testing/src/main/resources/logback.xml new file mode 100644 index 00000000..e80cf380 --- /dev/null +++ b/zulia-testing/src/main/resources/logback.xml @@ -0,0 +1,16 @@ + + + + true + + + %cyan(%date{ISO8601}) %gray([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n + + + + + + + + \ No newline at end of file From 6b3207645bb212b8bc485150f0b615ba17568e10 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Wed, 3 Apr 2024 11:03:52 -0400 Subject: [PATCH 03/12] refactor package names for zulia-testing project add start of stat facet config queries --- .../java/io/zulia/testing/TestRunner.java | 60 ++++++++++++++----- .../{ => config}/ConnectionConfig.java | 2 +- .../testing/{ => config}/FacetConfig.java | 2 +- .../testing/{ => config}/IndexConfig.java | 2 +- .../testing/{ => config}/QueryConfig.java | 15 ++++- .../zulia/testing/config/StatFacetConfig.java | 41 +++++++++++++ .../testing/{ => config}/TestConfig.java | 2 +- .../testing/{ => config}/ZuliaTestConfig.java | 2 +- .../testing/js/dto/FacetProxyObject.java | 6 +- ...{FacetValue.java => FacetValueObject.java} | 2 +- ...ueryResult.java => QueryResultObject.java} | 4 +- .../testing/{ => result}/TestResult.java | 4 +- 12 files changed, 113 insertions(+), 29 deletions(-) rename zulia-testing/src/main/java/io/zulia/testing/{ => config}/ConnectionConfig.java (94%) rename zulia-testing/src/main/java/io/zulia/testing/{ => config}/FacetConfig.java (92%) rename zulia-testing/src/main/java/io/zulia/testing/{ => config}/IndexConfig.java (94%) rename zulia-testing/src/main/java/io/zulia/testing/{ => config}/QueryConfig.java (74%) create mode 100644 zulia-testing/src/main/java/io/zulia/testing/config/StatFacetConfig.java rename zulia-testing/src/main/java/io/zulia/testing/{ => config}/TestConfig.java (89%) rename zulia-testing/src/main/java/io/zulia/testing/{ => config}/ZuliaTestConfig.java (97%) rename zulia-testing/src/main/java/io/zulia/testing/js/dto/{FacetValue.java => FacetValueObject.java} (70%) rename zulia-testing/src/main/java/io/zulia/testing/js/dto/{QueryResult.java => QueryResultObject.java} (75%) rename zulia-testing/src/main/java/io/zulia/testing/{ => result}/TestResult.java (89%) diff --git a/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java b/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java index f83c88cc..ba748563 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java +++ b/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java @@ -5,21 +5,31 @@ import io.zulia.client.command.builder.CountFacet; import io.zulia.client.command.builder.FilterQuery; import io.zulia.client.command.builder.Search; +import io.zulia.client.command.builder.StatFacet; import io.zulia.client.config.ZuliaPoolConfig; import io.zulia.client.pool.ZuliaWorkPool; import io.zulia.client.result.CompleteResult; import io.zulia.client.result.SearchResult; import io.zulia.message.ZuliaQuery; +import io.zulia.testing.config.ConnectionConfig; +import io.zulia.testing.config.FacetConfig; +import io.zulia.testing.config.IndexConfig; +import io.zulia.testing.config.QueryConfig; +import io.zulia.testing.config.StatFacetConfig; +import io.zulia.testing.config.TestConfig; +import io.zulia.testing.config.ZuliaTestConfig; import io.zulia.testing.js.dto.DocumentProxyObject; import io.zulia.testing.js.dto.FacetProxyObject; -import io.zulia.testing.js.dto.FacetValue; -import io.zulia.testing.js.dto.QueryResult; +import io.zulia.testing.js.dto.FacetValueObject; +import io.zulia.testing.js.dto.QueryResultObject; +import io.zulia.testing.result.TestResult; import io.zulia.util.exception.ThrowingSupplier; import org.bson.Document; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.HostAccess; import org.graalvm.polyglot.Value; import org.graalvm.polyglot.proxy.ProxyObject; +import org.jetbrains.annotations.NotNull; import org.yaml.snakeyaml.Yaml; import java.io.InputStream; @@ -33,18 +43,22 @@ public class TestRunner { public static void main(String[] args) throws Exception { Yaml yaml = new Yaml(); - ZuliaTestConfig dataQualityConfig; try (InputStream inputStream = TestRunner.class.getResource("/testing.yaml").openStream()) { - dataQualityConfig = yaml.loadAs(inputStream, ZuliaTestConfig.class); + List testResults = runTests(yaml, inputStream); + for (TestResult testResult : testResults) { + System.out.println(testResult.getTestId() + " result is " + testResult.isPassed()); + } } - List testResults = runTests(dataQualityConfig); - for (TestResult testResult : testResults) { - System.out.println(testResult.getTestId() + " result is " + testResult.isPassed()); - } + } + @NotNull + public static List runTests(Yaml yaml, InputStream inputStream) throws Exception { + ZuliaTestConfig zuliaTestConfig = yaml.loadAs(inputStream, ZuliaTestConfig.class); + return runTests(zuliaTestConfig); } + @NotNull public static List runTests(ZuliaTestConfig zuliaTestConfig) throws Exception { Map> connectionToConnectionConfig = new HashMap<>(); for (Map.Entry indexConfigEntry : zuliaTestConfig.getConnections().entrySet()) { @@ -67,7 +81,7 @@ public static List runTests(ZuliaTestConfig zuliaTestConfig) throws } } - Map resultMap = new HashMap<>(); + Map resultMap = new HashMap<>(); for (Map.Entry queryConfigEntry : zuliaTestConfig.getQueries().entrySet()) { String queryId = queryConfigEntry.getKey(); QueryConfig queryConfig = queryConfigEntry.getValue(); @@ -89,8 +103,19 @@ public static List runTests(ZuliaTestConfig zuliaTestConfig) throws } } + if (queryConfig.getStatFacets() != null) { + for (StatFacetConfig statFacetConfig : queryConfig.getStatFacets()) { + StatFacet statFacet = new StatFacet(statFacetConfig.getNumericField(), statFacetConfig.getFacetField()); + if (statFacetConfig.getTopN() != 0) { + statFacet.setTopN(statFacetConfig.getTopN()); + } + s.addStat(statFacet); + } + } + SearchResult searchResult = zuliaWorkPool.search(s); - QueryResult result = new QueryResult(); + + QueryResultObject result = new QueryResultObject(); result.count = searchResult.getTotalHits(); List docs = new ArrayList<>(); @@ -98,14 +123,14 @@ public static List runTests(ZuliaTestConfig zuliaTestConfig) throws Document document = completeResult.getDocument(); docs.add(new DocumentProxyObject(document)); } - Map> facetResults = new HashMap<>(); + Map> facetResults = new HashMap<>(); if (queryConfig.getFacets() != null) { for (FacetConfig facet : queryConfig.getFacets()) { - List facetValues = new ArrayList<>(); + List facetValues = new ArrayList<>(); List facetCounts = searchResult.getFacetCounts(facet.getField()); for (ZuliaQuery.FacetCount facetCount : facetCounts) { - FacetValue fv = new FacetValue(); + FacetValueObject fv = new FacetValueObject(); fv.label = facetCount.getFacet(); fv.count = facetCount.getCount(); facetValues.add(fv); @@ -115,6 +140,12 @@ public static List runTests(ZuliaTestConfig zuliaTestConfig) throws result.facet = new FacetProxyObject(facetResults); } + if (queryConfig.getStatFacets() != null) { + for (StatFacetConfig statFacet : queryConfig.getStatFacets()) { + + } + } + result.doc = docs; resultMap.put(queryId, result); } @@ -122,7 +153,7 @@ public static List runTests(ZuliaTestConfig zuliaTestConfig) throws List testResults = new ArrayList<>(); try (Context context = Context.newBuilder("js").option("engine.WarnInterpreterOnly", "false").allowHostAccess(HostAccess.ALL).build()) { Value jsValue = context.getBindings("js"); - for (Map.Entry countToEntry : resultMap.entrySet()) { + for (Map.Entry countToEntry : resultMap.entrySet()) { jsValue.putMember(countToEntry.getKey(), countToEntry.getValue()); } @@ -133,7 +164,6 @@ public static List runTests(ZuliaTestConfig zuliaTestConfig) throws testResult.setTestId(testConfigEntry.getKey()); testResult.setTestConfig(testConfig); Value js = context.eval("js", testConfig.getExpr()); - System.out.println(js); testResult.setPassed(js.asBoolean()); testResults.add(testResult); } diff --git a/zulia-testing/src/main/java/io/zulia/testing/ConnectionConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java similarity index 94% rename from zulia-testing/src/main/java/io/zulia/testing/ConnectionConfig.java rename to zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java index 1ab93c30..5285849d 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/ConnectionConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java @@ -1,4 +1,4 @@ -package io.zulia.testing; +package io.zulia.testing.config; public class ConnectionConfig { diff --git a/zulia-testing/src/main/java/io/zulia/testing/FacetConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/FacetConfig.java similarity index 92% rename from zulia-testing/src/main/java/io/zulia/testing/FacetConfig.java rename to zulia-testing/src/main/java/io/zulia/testing/config/FacetConfig.java index 012eed42..bf6e490a 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/FacetConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/FacetConfig.java @@ -1,4 +1,4 @@ -package io.zulia.testing; +package io.zulia.testing.config; public class FacetConfig { diff --git a/zulia-testing/src/main/java/io/zulia/testing/IndexConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/IndexConfig.java similarity index 94% rename from zulia-testing/src/main/java/io/zulia/testing/IndexConfig.java rename to zulia-testing/src/main/java/io/zulia/testing/config/IndexConfig.java index 11d07785..1b84fd60 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/IndexConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/IndexConfig.java @@ -1,4 +1,4 @@ -package io.zulia.testing; +package io.zulia.testing.config; public final class IndexConfig { diff --git a/zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java similarity index 74% rename from zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java rename to zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java index af1ee5f0..23f0a2f3 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/QueryConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java @@ -1,4 +1,4 @@ -package io.zulia.testing; +package io.zulia.testing.config; import java.util.List; @@ -10,6 +10,8 @@ public final class QueryConfig { private List facets; + private List statFacets; + public QueryConfig() { } @@ -46,8 +48,17 @@ public void setFacets(List facets) { this.facets = facets; } + public List getStatFacets() { + return statFacets; + } + + public void setStatFacets(List statFacets) { + this.statFacets = statFacets; + } + @Override public String toString() { - return "QueryConfig{" + "index='" + index + '\'' + ", query='" + query + '\'' + ", amount=" + amount + ", facets=" + facets + '}'; + return "QueryConfig{" + "index='" + index + '\'' + ", query='" + query + '\'' + ", amount=" + amount + ", facets=" + facets + ", statFacets=" + + statFacets + '}'; } } diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/StatFacetConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/StatFacetConfig.java new file mode 100644 index 00000000..a7af6f92 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/config/StatFacetConfig.java @@ -0,0 +1,41 @@ +package io.zulia.testing.config; + +public class StatFacetConfig { + + private String facetField; + + private String numericField; + private int topN; + + public StatFacetConfig() { + } + + public String getFacetField() { + return facetField; + } + + public void setFacetField(String facetField) { + this.facetField = facetField; + } + + public String getNumericField() { + return numericField; + } + + public void setNumericField(String numericField) { + this.numericField = numericField; + } + + public int getTopN() { + return topN; + } + + public void setTopN(int topN) { + this.topN = topN; + } + + @Override + public String toString() { + return "StatFacetConfig{" + "facetField='" + facetField + '\'' + ", numericField='" + numericField + '\'' + ", topN=" + topN + '}'; + } +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/TestConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/TestConfig.java similarity index 89% rename from zulia-testing/src/main/java/io/zulia/testing/TestConfig.java rename to zulia-testing/src/main/java/io/zulia/testing/config/TestConfig.java index e8d4ef2c..455ea466 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/TestConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/TestConfig.java @@ -1,4 +1,4 @@ -package io.zulia.testing; +package io.zulia.testing.config; public final class TestConfig { private String expr; diff --git a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java similarity index 97% rename from zulia-testing/src/main/java/io/zulia/testing/ZuliaTestConfig.java rename to zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java index dcaafd4b..b72abb26 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java @@ -1,4 +1,4 @@ -package io.zulia.testing; +package io.zulia.testing.config; import java.util.Map; diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java index 6bb58466..df5233f1 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java @@ -8,9 +8,9 @@ import java.util.Map; public class FacetProxyObject implements ProxyObject { - private final Map> keyToFacets; + private final Map> keyToFacets; - public FacetProxyObject(Map> keyToFacets) { + public FacetProxyObject(Map> keyToFacets) { this.keyToFacets = keyToFacets; } @@ -47,7 +47,7 @@ public Object get(long index) { } @Override - public List getMember(String key) { + public List getMember(String key) { return keyToFacets.get(key); } diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValue.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValueObject.java similarity index 70% rename from zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValue.java rename to zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValueObject.java index 7361422c..9e3883eb 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValue.java +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetValueObject.java @@ -1,6 +1,6 @@ package io.zulia.testing.js.dto; -public class FacetValue { +public class FacetValueObject { public String label; public long count; diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResult.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java similarity index 75% rename from zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResult.java rename to zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java index bbed5e96..1d8d1303 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResult.java +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java @@ -4,9 +4,9 @@ import java.util.List; -public class QueryResult { +public class QueryResultObject { - public QueryResult() { + public QueryResultObject() { } diff --git a/zulia-testing/src/main/java/io/zulia/testing/TestResult.java b/zulia-testing/src/main/java/io/zulia/testing/result/TestResult.java similarity index 89% rename from zulia-testing/src/main/java/io/zulia/testing/TestResult.java rename to zulia-testing/src/main/java/io/zulia/testing/result/TestResult.java index 1ceed420..05f52004 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/TestResult.java +++ b/zulia-testing/src/main/java/io/zulia/testing/result/TestResult.java @@ -1,4 +1,6 @@ -package io.zulia.testing; +package io.zulia.testing.result; + +import io.zulia.testing.config.TestConfig; public class TestResult { From 161a52825a4b9093429b55fefdaeb82839192362 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Thu, 4 Apr 2024 09:50:46 -0400 Subject: [PATCH 04/12] bump common compress to 1.26.1 (from 1.25.0) for security fix bump flapdoodle mongo to 4.12.2 (from 4.11.1) bump jansi to 2.4.1 (from 2.4.0) --- gradle/libs.versions.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5806267d..c4aacf72 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,9 +15,9 @@ annontations = { module = "org.jetbrains:annotations", version = "24.0.0" } awssdk-s3 = { module = "software.amazon.awssdk:s3" } bson = { module = "org.mongodb:bson", version.ref = "mongodb" } caffeine = { module = "com.github.ben-manes.caffeine:caffeine" } -commons-compress = { module = "org.apache.commons:commons-compress", version = "1.25.0" } +commons-compress = { module = "org.apache.commons:commons-compress", version = "1.26.1" } commons-pool2 = { module = "org.apache.commons:commons-pool2", version = "2.11.1" } -flapdoodle-mongo = { module = "de.flapdoodle.embed:de.flapdoodle.embed.mongo", version = "4.11.1" } +flapdoodle-mongo = { module = "de.flapdoodle.embed:de.flapdoodle.embed.mongo", version = "4.12.2" } guava = { module = "com.google.guava:guava", version = "32.1.3-jre" } grpc-netty-shaded = { module = "io.grpc:grpc-netty-shaded", version.ref = "grpc" } grpc-protobuf = { module = "io.grpc:grpc-protobuf", version.ref = "grpc" } @@ -26,7 +26,7 @@ graalvm-js = { module = "org.graalvm.js:js", version = "23.0.3" } gson = { module = "com.google.code.gson:gson", version = "2.10.1" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version = "2.16.1" } jakarta-validation = { module = "jakarta.validation:jakarta.validation-api" } -jansi = { module = "org.fusesource.jansi:jansi", version = "2.4.0" } +jansi = { module = "org.fusesource.jansi:jansi", version = "2.4.1" } javax-annotation = { module = "javax.annotation:javax.annotation-api", version = "1.3.2" } json-io = { module = "com.cedarsoftware:json-io", version = "4.19.1" } junit-bom = { module = "org.junit:junit-bom", version = "5.10.1" } From 19f5e2719032dd1def49aed86ef42432bdf71ff5 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Sat, 6 Apr 2024 15:02:37 -0400 Subject: [PATCH 05/12] refactor zulia command shared args into separate package split off zulia-tools include new zuliatest in zulia-tools --- settings.gradle | 2 +- zulia-cmd-shared/build.gradle.kts | 13 + .../common/SelectiveStackTraceHandler.java | 2 +- .../io/zulia}/cmd/common/ShowStackArgs.java | 2 +- .../io/zulia/cmd/common}/ZuliaCommonCmd.java | 3 +- .../cmd/common/ZuliaVersionProvider.java | 2 +- .../src/main/resources/logback.xml | 0 .../src/test/resources/sample_testing.yaml | 18 +- zulia-server/build.gradle.kts | 170 +------------ .../main/java/io/zulia/server/cmd/ZuliaD.java | 7 +- .../zulia/server/cmd/zuliad/StartNodeCmd.java | 2 +- .../zulia/server/cmd/zuliad/ZuliaDConfig.java | 2 +- zulia-testing/build.gradle.kts | 1 - .../java/io/zulia/testing/TestRunner.java | 174 -------------- .../io/zulia/testing/ZuliaTestRunner.java | 224 ++++++++++++++++++ .../zulia/testing/config/ZuliaTestConfig.java | 32 ++- ...yObject.java => KeyedListProxyObject.java} | 8 +- .../testing/js/dto/QueryResultObject.java | 5 +- .../testing/js/dto/StatFacetValueObject.java | 11 + zulia-testing/src/main/resources/logback.xml | 16 -- .../src/test/resources/sample_testing.yaml | 51 ++++ zulia-tools/build.gradle.kts | 210 ++++++++++++++++ .../main/java/io/zulia/tools}/cmd/Zulia.java | 11 +- .../java/io/zulia/tools}/cmd/ZuliaAdmin.java | 9 +- .../java/io/zulia/tools}/cmd/ZuliaDump.java | 13 +- .../java/io/zulia/tools}/cmd/ZuliaExport.java | 16 +- .../java/io/zulia/tools}/cmd/ZuliaImport.java | 15 +- .../io/zulia/tools}/cmd/ZuliaRestore.java | 13 +- .../java/io/zulia/tools/cmd/ZuliaTest.java | 68 ++++++ .../io/zulia/tools}/cmd/common/AliasArgs.java | 2 +- .../tools}/cmd/common/MultipleIndexArgs.java | 2 +- .../tools}/cmd/common/SingleIndexArgs.java | 2 +- .../zulia/tools}/cmd/common/ThreadedArgs.java | 2 +- .../zulia/tools}/cmd/common/ZuliaCmdUtil.java | 2 +- .../tools}/cmd/zuliaadmin/ClearIndexCmd.java | 6 +- .../tools}/cmd/zuliaadmin/ConnectionInfo.java | 2 +- .../tools}/cmd/zuliaadmin/CreateAliasCmd.java | 4 +- .../tools}/cmd/zuliaadmin/DeleteAliasCmd.java | 6 +- .../tools}/cmd/zuliaadmin/DeleteIndexCmd.java | 6 +- .../cmd/zuliaadmin/DisplayAliasesCmd.java | 6 +- .../cmd/zuliaadmin/DisplayIndexesCmd.java | 6 +- .../cmd/zuliaadmin/DisplayNodesCmd.java | 6 +- .../tools}/cmd/zuliaadmin/DocCountCmd.java | 8 +- .../cmd/zuliaadmin/ExportAliasesCmd.java | 6 +- .../zuliaadmin/FetchAssociatedFileCmd.java | 6 +- .../cmd/zuliaadmin/ImportAliasesCmd.java | 4 +- .../cmd/zuliaadmin/OptimizeIndexCmd.java | 6 +- .../tools}/cmd/zuliaadmin/ReindexCmd.java | 6 +- .../zuliaadmin/StoreAssociatedFileCmd.java | 6 +- .../src/test/resources/sample_testing.yaml | 42 ++++ 50 files changed, 767 insertions(+), 469 deletions(-) create mode 100644 zulia-cmd-shared/build.gradle.kts rename {zulia-server/src/main/java/io/zulia/server => zulia-cmd-shared/src/main/java/io/zulia}/cmd/common/SelectiveStackTraceHandler.java (94%) rename {zulia-server/src/main/java/io/zulia/server => zulia-cmd-shared/src/main/java/io/zulia}/cmd/common/ShowStackArgs.java (88%) rename {zulia-server/src/main/java/io/zulia/server/cmd => zulia-cmd-shared/src/main/java/io/zulia/cmd/common}/ZuliaCommonCmd.java (94%) rename {zulia-server/src/main/java/io/zulia/server => zulia-cmd-shared/src/main/java/io/zulia}/cmd/common/ZuliaVersionProvider.java (87%) rename {zulia-server => zulia-cmd-shared}/src/main/resources/logback.xml (100%) rename zulia-testing/src/main/resources/testing.yaml => zulia-cmd-shared/src/test/resources/sample_testing.yaml (61%) delete mode 100644 zulia-testing/src/main/java/io/zulia/testing/TestRunner.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java rename zulia-testing/src/main/java/io/zulia/testing/js/dto/{FacetProxyObject.java => KeyedListProxyObject.java} (79%) create mode 100644 zulia-testing/src/main/java/io/zulia/testing/js/dto/StatFacetValueObject.java delete mode 100644 zulia-testing/src/main/resources/logback.xml create mode 100644 zulia-testing/src/test/resources/sample_testing.yaml create mode 100644 zulia-tools/build.gradle.kts rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/Zulia.java (98%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/ZuliaAdmin.java (82%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/ZuliaDump.java (93%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/ZuliaExport.java (90%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/ZuliaImport.java (85%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/ZuliaRestore.java (94%) create mode 100644 zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/common/AliasArgs.java (97%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/common/MultipleIndexArgs.java (96%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/common/SingleIndexArgs.java (86%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/common/ThreadedArgs.java (88%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/common/ZuliaCmdUtil.java (99%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/ClearIndexCmd.java (84%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/ConnectionInfo.java (96%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/CreateAliasCmd.java (91%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/DeleteAliasCmd.java (85%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/DeleteIndexCmd.java (87%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/DisplayAliasesCmd.java (87%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/DisplayIndexesCmd.java (94%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/DisplayNodesCmd.java (92%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/DocCountCmd.java (92%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/ExportAliasesCmd.java (91%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/FetchAssociatedFileCmd.java (90%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/ImportAliasesCmd.java (93%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/OptimizeIndexCmd.java (84%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/ReindexCmd.java (85%) rename {zulia-server/src/main/java/io/zulia/server => zulia-tools/src/main/java/io/zulia/tools}/cmd/zuliaadmin/StoreAssociatedFileCmd.java (91%) create mode 100644 zulia-tools/src/test/resources/sample_testing.yaml diff --git a/settings.gradle b/settings.gradle index 44c1c14e..472aac55 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,5 @@ rootProject.name = "ZuliaSearch" -include 'zulia-analyzer', 'zulia-common', 'zulia-client', 'zulia-data', 'zulia-query-parser', 'zulia-server', 'zulia-testing', 'zulia-util' +include 'zulia-analyzer', 'zulia-common', 'zulia-cmd-shared', 'zulia-client', 'zulia-data', 'zulia-query-parser', 'zulia-server', 'zulia-testing', 'zulia-tools', 'zulia-util' diff --git a/zulia-cmd-shared/build.gradle.kts b/zulia-cmd-shared/build.gradle.kts new file mode 100644 index 00000000..734849eb --- /dev/null +++ b/zulia-cmd-shared/build.gradle.kts @@ -0,0 +1,13 @@ +plugins { + `java-library` +} + +description = "Zulia CMD Shared" + +dependencies { + annotationProcessor(libs.picocli.codegen) + api(project(":zulia-common")) + implementation(libs.picocli.base) +} + + diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/common/SelectiveStackTraceHandler.java b/zulia-cmd-shared/src/main/java/io/zulia/cmd/common/SelectiveStackTraceHandler.java similarity index 94% rename from zulia-server/src/main/java/io/zulia/server/cmd/common/SelectiveStackTraceHandler.java rename to zulia-cmd-shared/src/main/java/io/zulia/cmd/common/SelectiveStackTraceHandler.java index dbbd882b..5fa2301c 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/common/SelectiveStackTraceHandler.java +++ b/zulia-cmd-shared/src/main/java/io/zulia/cmd/common/SelectiveStackTraceHandler.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd.common; +package io.zulia.cmd.common; import picocli.CommandLine; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/common/ShowStackArgs.java b/zulia-cmd-shared/src/main/java/io/zulia/cmd/common/ShowStackArgs.java similarity index 88% rename from zulia-server/src/main/java/io/zulia/server/cmd/common/ShowStackArgs.java rename to zulia-cmd-shared/src/main/java/io/zulia/cmd/common/ShowStackArgs.java index 1caef6be..4a0b4876 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/common/ShowStackArgs.java +++ b/zulia-cmd-shared/src/main/java/io/zulia/cmd/common/ShowStackArgs.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd.common; +package io.zulia.cmd.common; import picocli.CommandLine; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaCommonCmd.java b/zulia-cmd-shared/src/main/java/io/zulia/cmd/common/ZuliaCommonCmd.java similarity index 94% rename from zulia-server/src/main/java/io/zulia/server/cmd/ZuliaCommonCmd.java rename to zulia-cmd-shared/src/main/java/io/zulia/cmd/common/ZuliaCommonCmd.java index 7febafc9..c00f74af 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaCommonCmd.java +++ b/zulia-cmd-shared/src/main/java/io/zulia/cmd/common/ZuliaCommonCmd.java @@ -1,6 +1,5 @@ -package io.zulia.server.cmd; +package io.zulia.cmd.common; -import io.zulia.server.cmd.common.SelectiveStackTraceHandler; import picocli.CommandLine; import picocli.CommandLine.Help.Ansi.Style; import picocli.CommandLine.Help.ColorScheme; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/common/ZuliaVersionProvider.java b/zulia-cmd-shared/src/main/java/io/zulia/cmd/common/ZuliaVersionProvider.java similarity index 87% rename from zulia-server/src/main/java/io/zulia/server/cmd/common/ZuliaVersionProvider.java rename to zulia-cmd-shared/src/main/java/io/zulia/cmd/common/ZuliaVersionProvider.java index a626335a..14c8e84d 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/common/ZuliaVersionProvider.java +++ b/zulia-cmd-shared/src/main/java/io/zulia/cmd/common/ZuliaVersionProvider.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd.common; +package io.zulia.cmd.common; import io.zulia.util.ZuliaVersion; import picocli.CommandLine; diff --git a/zulia-server/src/main/resources/logback.xml b/zulia-cmd-shared/src/main/resources/logback.xml similarity index 100% rename from zulia-server/src/main/resources/logback.xml rename to zulia-cmd-shared/src/main/resources/logback.xml diff --git a/zulia-testing/src/main/resources/testing.yaml b/zulia-cmd-shared/src/test/resources/sample_testing.yaml similarity index 61% rename from zulia-testing/src/main/resources/testing.yaml rename to zulia-cmd-shared/src/test/resources/sample_testing.yaml index 45b46a24..cee9625e 100644 --- a/zulia-testing/src/main/resources/testing.yaml +++ b/zulia-cmd-shared/src/test/resources/sample_testing.yaml @@ -4,7 +4,7 @@ connections: port: 32191 indexes: publications: - indexName: publications_2024 + indexName: publications connection: testConnection queries: pubsStar: @@ -15,26 +15,28 @@ queries: query: "-title:*" pubsTest: index: publications - query: "-title:*" + query: "title:*" amount: 1 - facet: + facets: - field: pubYear topN: 10 - field: journalName topN: 10 - - + statFacets: + - facetField: pubYear + numericField: rcr + topN: 10 tests: pubsCount: - expr: pubsStar.count > 30000000 + expr: pubsStar.count > 200000 pubsWithTitleCount: expr: pubsWithoutTitle.count < pubsStar.count * 0.01 test1: expr: pubsTest.count > 1 test2: - expr: pubsTest.doc[0]["title"].length == 0 + expr: pubsTest.doc[0]["authors"][0]["lastName"] == "Abumere" test3: - expr: pubsTest.facet["pubYear"][1] > 1000000 + expr: pubsTest.facet["pubYear"][0].label == "2022" && pubsTest.facet["pubYear"][0].count > 1000000 diff --git a/zulia-server/build.gradle.kts b/zulia-server/build.gradle.kts index e6aff356..7ab6e471 100644 --- a/zulia-server/build.gradle.kts +++ b/zulia-server/build.gradle.kts @@ -37,8 +37,8 @@ tasks.withType { } dependencies { - implementation(project(":zulia-client")) //needed for admin tools implementation(project(":zulia-query-parser")) + implementation(project(":zulia-cmd-shared")) annotationProcessor(libs.micronaut.http.validation) annotationProcessor(libs.micronaut.openapi) annotationProcessor(libs.micronaut.serde.processor) @@ -73,100 +73,26 @@ dependencies { testImplementation(libs.flapdoodle.mongo) testImplementation(libs.micronaut.http.client) testImplementation(libs.micronaut.test.junit5) + testImplementation(project(":zulia-client")) } tasks.withType { options.isFork = true options.forkOptions.jvmArgs?.addAll(listOf("-Dmicronaut.openapi.views.spec=swagger-ui.enabled=true,swagger-ui.theme=flattop")) } -val zuliaScriptTask = tasks.getByName("startScripts") - -zuliaScriptTask.applicationName = "zulia" -zuliaScriptTask.mainClass.set("io.zulia.server.cmd.Zulia") -val zuliaAdminScriptTask = tasks.register("createZuliaAdminScript") { - applicationName = "zuliaadmin" - mainClass.set("io.zulia.server.cmd.ZuliaAdmin") - outputDir = zuliaScriptTask.outputDir - classpath = zuliaScriptTask.classpath - - doLast { - val unixScriptFile = file(unixScript) - val text = unixScriptFile.readText(Charsets.UTF_8) - val newText = text.replace("APP_HOME=\"`pwd -P`\"", "export APP_HOME=\"`pwd -P`\"") - unixScriptFile.writeText(newText, Charsets.UTF_8) - } -} - -val zuliaDScriptTask = tasks.register("createZuliaDScript") { +val zuliaScriptTask = tasks.getByName("startScripts") { applicationName = "zuliad" mainClass.set("io.zulia.server.cmd.ZuliaD") - outputDir = zuliaScriptTask.outputDir - classpath = zuliaScriptTask.classpath - doLast { val unixScriptFile = file(unixScript) val text = unixScriptFile.readText(Charsets.UTF_8) - val newText = text.replace("APP_HOME=\"`pwd -P`\"", "export APP_HOME=\"`pwd -P`\"") + val newText = text.replace("exec ", "export APP_HOME\nexec ") unixScriptFile.writeText(newText, Charsets.UTF_8) } } -val zuliaDumpScriptTask = tasks.register("createZuliaDumpScript") { - applicationName = "zuliadump" - mainClass.set("io.zulia.server.cmd.ZuliaDump") - outputDir = zuliaScriptTask.outputDir - classpath = zuliaScriptTask.classpath - doLast { - val unixScriptFile = file(unixScript) - val text = unixScriptFile.readText(Charsets.UTF_8) - val newText = text.replace("APP_HOME=\"`pwd -P`\"", "export APP_HOME=\"`pwd -P`\"") - unixScriptFile.writeText(newText, Charsets.UTF_8) - } -} - -val zuliaRestoreScriptTask = tasks.register("createZuliaRestoreScript") { - applicationName = "zuliarestore" - mainClass.set("io.zulia.server.cmd.ZuliaRestore") - outputDir = zuliaScriptTask.outputDir - classpath = zuliaScriptTask.classpath - - doLast { - val unixScriptFile = file(unixScript) - val text = unixScriptFile.readText(Charsets.UTF_8) - val newText = text.replace("APP_HOME=\"`pwd -P`\"", "export APP_HOME=\"`pwd -P`\"") - unixScriptFile.writeText(newText, Charsets.UTF_8) - } -} - -val zuliaExportScriptTask = tasks.register("createZuliaExportScript") { - applicationName = "zuliaexport" - mainClass.set("io.zulia.server.cmd.ZuliaExport") - outputDir = zuliaScriptTask.outputDir - classpath = zuliaScriptTask.classpath - - doLast { - val unixScriptFile = file(unixScript) - val text = unixScriptFile.readText(Charsets.UTF_8) - val newText = text.replace("APP_HOME=\"`pwd -P`\"", "export APP_HOME=\"`pwd -P`\"") - unixScriptFile.writeText(newText, Charsets.UTF_8) - } -} - -val zuliaImportScriptTask = tasks.register("createZuliaImportScript") { - applicationName = "zuliaimport" - mainClass.set("io.zulia.server.cmd.ZuliaImport") - outputDir = zuliaScriptTask.outputDir - classpath = zuliaScriptTask.classpath - - doLast { - val unixScriptFile = file(unixScript) - val text = unixScriptFile.readText(Charsets.UTF_8) - val newText = text.replace("APP_HOME=\"`pwd -P`\"", "export APP_HOME=\"`pwd -P`\"") - unixScriptFile.writeText(newText, Charsets.UTF_8) - } -} tasks.register("autocompleteDir") { doLast { @@ -174,88 +100,22 @@ tasks.register("autocompleteDir") { } } -task("picoCliZuliaAutoComplete", JavaExec::class) { - dependsOn("autocompleteDir") - mainClass.set("picocli.AutoComplete") - classpath = sourceSets["main"].runtimeClasspath - args = listOf("--force", "--completionScript", "${layout.buildDirectory.get()}/autocomplete/zulia.sh", "io.zulia.server.cmd.Zulia") -} - task("picoCliZuliaDAutoComplete", JavaExec::class) { - dependsOn("autocompleteDir") - mainClass.set("picocli.AutoComplete") - classpath = sourceSets["main"].runtimeClasspath - args = listOf("--force", "--completionScript", "${layout.buildDirectory.get()}/autocomplete/zuliad.sh", "io.zulia.server.cmd.ZuliaD") -} - -task("picoCliZuliaAdminAutoComplete", JavaExec::class) { dependsOn("autocompleteDir") mainClass.set("picocli.AutoComplete") classpath = sourceSets["main"].runtimeClasspath args = listOf( "--force", "--completionScript", - "${layout.buildDirectory.get()}/autocomplete/zuliaadmin.sh", - "io.zulia.server.cmd.ZuliaAdmin" + "${layout.buildDirectory.get()}/autocomplete/zuliad.sh", + "io.zulia.server.cmd.ZuliaD" ) } -task("picoCliZuliaDumpAutoComplete", JavaExec::class) { - dependsOn("autocompleteDir") - mainClass.set("picocli.AutoComplete") - classpath = sourceSets["main"].runtimeClasspath - args = - listOf("--force", "--completionScript", "${layout.buildDirectory.get()}/autocomplete/zuliadump.sh", "io.zulia.server.cmd.ZuliaDump") -} - - - -task("picoCliZuliaRestoreAutoComplete", JavaExec::class) { - dependsOn("autocompleteDir") - mainClass.set("picocli.AutoComplete") - classpath = sourceSets["main"].runtimeClasspath - args = listOf( - "--force", - "--completionScript", - "${layout.buildDirectory.get()}/autocomplete/zuliarestore.sh", - "io.zulia.server.cmd.ZuliaRestore" - ) -} - - -task("picoCliZuliaImportAutoComplete", JavaExec::class) { - dependsOn("autocompleteDir") - mainClass.set("picocli.AutoComplete") - classpath = sourceSets["main"].runtimeClasspath - args = listOf( - "--force", - "--completionScript", - "${layout.buildDirectory.get()}/autocomplete/zuliaimport.sh", - "io.zulia.server.cmd.ZuliaImport" - ) -} - -task("picoCliZuliaExportAutoComplete", JavaExec::class) { - dependsOn("autocompleteDir") - mainClass.set("picocli.AutoComplete") - classpath = sourceSets["main"].runtimeClasspath - args = listOf( - "--force", - "--completionScript", - "${layout.buildDirectory.get()}/autocomplete/zuliaexport.sh", - "io.zulia.server.cmd.ZuliaExport" - ) -} tasks.withType { dependsOn( - "picoCliZuliaAutoComplete", "picoCliZuliaDAutoComplete", - "picoCliZuliaAdminAutoComplete", - "picoCliZuliaDumpAutoComplete", - "picoCliZuliaRestoreAutoComplete", - "picoCliZuliaImportAutoComplete", - "picoCliZuliaExportAutoComplete" ) } @@ -263,24 +123,6 @@ tasks.withType { distributions { main { contents { - from(zuliaAdminScriptTask) { - into("bin") - } - from(zuliaDScriptTask) { - into("bin") - } - from(zuliaDumpScriptTask) { - into("bin") - } - from(zuliaRestoreScriptTask) { - into("bin") - } - from(zuliaExportScriptTask) { - into("bin") - } - from(zuliaImportScriptTask) { - into("bin") - } from("${layout.buildDirectory.get()}/autocomplete/") { into("bin/autocomplete") } diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaD.java b/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaD.java index abc9c62e..ff269554 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaD.java +++ b/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaD.java @@ -1,7 +1,8 @@ package io.zulia.server.cmd; -import io.zulia.server.cmd.common.ShowStackArgs; -import io.zulia.server.cmd.common.ZuliaVersionProvider; +import io.zulia.cmd.common.ShowStackArgs; +import io.zulia.cmd.common.ZuliaCommonCmd; +import io.zulia.cmd.common.ZuliaVersionProvider; import io.zulia.server.cmd.zuliad.AddNodeCmd; import io.zulia.server.cmd.zuliad.ListNodesCmd; import io.zulia.server.cmd.zuliad.RemoveNodeCmd; @@ -32,7 +33,7 @@ public String getConfigPath() { } public static void main(String[] args) { - + System.out.println("APP_HOME is set to " + System.getenv("APP_HOME")); ZuliaCommonCmd.runCommandLine(new ZuliaD(), args); } diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliad/StartNodeCmd.java b/zulia-server/src/main/java/io/zulia/server/cmd/zuliad/StartNodeCmd.java index 105f4ac5..b044377a 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliad/StartNodeCmd.java +++ b/zulia-server/src/main/java/io/zulia/server/cmd/zuliad/StartNodeCmd.java @@ -1,7 +1,7 @@ package io.zulia.server.cmd.zuliad; +import io.zulia.cmd.common.ZuliaCommonCmd; import io.zulia.message.ZuliaBase; -import io.zulia.server.cmd.ZuliaCommonCmd; import io.zulia.server.cmd.ZuliaD; import io.zulia.server.config.NodeService; import io.zulia.server.config.ZuliaConfig; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliad/ZuliaDConfig.java b/zulia-server/src/main/java/io/zulia/server/cmd/zuliad/ZuliaDConfig.java index 5b0e9f1a..a68255c3 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliad/ZuliaDConfig.java +++ b/zulia-server/src/main/java/io/zulia/server/cmd/zuliad/ZuliaDConfig.java @@ -12,8 +12,8 @@ import com.mongodb.ServerApiVersion; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; +import io.zulia.cmd.common.ZuliaCommonCmd; import io.zulia.message.ZuliaBase; -import io.zulia.server.cmd.ZuliaCommonCmd; import io.zulia.server.config.NodeService; import io.zulia.server.config.ZuliaConfig; import io.zulia.server.config.cluster.MongoAuth; diff --git a/zulia-testing/build.gradle.kts b/zulia-testing/build.gradle.kts index bcdc4752..ff1372d8 100644 --- a/zulia-testing/build.gradle.kts +++ b/zulia-testing/build.gradle.kts @@ -8,7 +8,6 @@ description = "Zulia Testing" dependencies { api(project(":zulia-util")) api(project(":zulia-client")) - implementation(libs.snake.yaml) implementation(libs.graalvm.js) implementation(libs.logback.classic) implementation(libs.jansi) diff --git a/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java b/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java deleted file mode 100644 index ba748563..00000000 --- a/zulia-testing/src/main/java/io/zulia/testing/TestRunner.java +++ /dev/null @@ -1,174 +0,0 @@ -package io.zulia.testing; - -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import io.zulia.client.command.builder.CountFacet; -import io.zulia.client.command.builder.FilterQuery; -import io.zulia.client.command.builder.Search; -import io.zulia.client.command.builder.StatFacet; -import io.zulia.client.config.ZuliaPoolConfig; -import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.client.result.CompleteResult; -import io.zulia.client.result.SearchResult; -import io.zulia.message.ZuliaQuery; -import io.zulia.testing.config.ConnectionConfig; -import io.zulia.testing.config.FacetConfig; -import io.zulia.testing.config.IndexConfig; -import io.zulia.testing.config.QueryConfig; -import io.zulia.testing.config.StatFacetConfig; -import io.zulia.testing.config.TestConfig; -import io.zulia.testing.config.ZuliaTestConfig; -import io.zulia.testing.js.dto.DocumentProxyObject; -import io.zulia.testing.js.dto.FacetProxyObject; -import io.zulia.testing.js.dto.FacetValueObject; -import io.zulia.testing.js.dto.QueryResultObject; -import io.zulia.testing.result.TestResult; -import io.zulia.util.exception.ThrowingSupplier; -import org.bson.Document; -import org.graalvm.polyglot.Context; -import org.graalvm.polyglot.HostAccess; -import org.graalvm.polyglot.Value; -import org.graalvm.polyglot.proxy.ProxyObject; -import org.jetbrains.annotations.NotNull; -import org.yaml.snakeyaml.Yaml; - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class TestRunner { - - public static void main(String[] args) throws Exception { - Yaml yaml = new Yaml(); - - try (InputStream inputStream = TestRunner.class.getResource("/testing.yaml").openStream()) { - List testResults = runTests(yaml, inputStream); - for (TestResult testResult : testResults) { - System.out.println(testResult.getTestId() + " result is " + testResult.isPassed()); - } - } - - } - - @NotNull - public static List runTests(Yaml yaml, InputStream inputStream) throws Exception { - ZuliaTestConfig zuliaTestConfig = yaml.loadAs(inputStream, ZuliaTestConfig.class); - return runTests(zuliaTestConfig); - } - - @NotNull - public static List runTests(ZuliaTestConfig zuliaTestConfig) throws Exception { - Map> connectionToConnectionConfig = new HashMap<>(); - for (Map.Entry indexConfigEntry : zuliaTestConfig.getConnections().entrySet()) { - ConnectionConfig connectionConfig = indexConfigEntry.getValue(); - - ThrowingSupplier throwingSupplier = () -> { - ZuliaPoolConfig zuliaPoolConfig = new ZuliaPoolConfig().addNode(connectionConfig.getServerAddress(), connectionConfig.getPort()); - return new ZuliaWorkPool(zuliaPoolConfig); - }; - - connectionToConnectionConfig.put(indexConfigEntry.getKey(), Suppliers.memoize(throwingSupplier)); - } - - for (Map.Entry indexConfigEntry : zuliaTestConfig.getIndexes().entrySet()) { - IndexConfig indexConfig = indexConfigEntry.getValue(); - Supplier zuliaWorkPoolSupplier = connectionToConnectionConfig.get(indexConfig.getConnection()); - if (zuliaWorkPoolSupplier == null) { - throw new IllegalArgumentException( - "Failed to find connection config <" + indexConfig.getConnection() + "> for query <" + indexConfigEntry.getKey() + ">"); - } - } - - Map resultMap = new HashMap<>(); - for (Map.Entry queryConfigEntry : zuliaTestConfig.getQueries().entrySet()) { - String queryId = queryConfigEntry.getKey(); - QueryConfig queryConfig = queryConfigEntry.getValue(); - IndexConfig indexConfig = zuliaTestConfig.getIndexes().get(queryConfig.getIndex()); - Supplier zuliaWorkPoolSupplier = connectionToConnectionConfig.get(indexConfig.getConnection()); - ZuliaWorkPool zuliaWorkPool = zuliaWorkPoolSupplier.get(); - - Search s = new Search(indexConfig.getIndexName()); - s.addQuery(new FilterQuery(queryConfig.getQuery())); - s.setAmount(queryConfig.getAmount()); - - if (queryConfig.getFacets() != null) { - for (FacetConfig facet : queryConfig.getFacets()) { - CountFacet countFacet = new CountFacet(facet.getField()); - if (facet.getTopN() != 0) { - countFacet.setTopN(facet.getTopN()); - } - s.addCountFacet(countFacet); - } - } - - if (queryConfig.getStatFacets() != null) { - for (StatFacetConfig statFacetConfig : queryConfig.getStatFacets()) { - StatFacet statFacet = new StatFacet(statFacetConfig.getNumericField(), statFacetConfig.getFacetField()); - if (statFacetConfig.getTopN() != 0) { - statFacet.setTopN(statFacetConfig.getTopN()); - } - s.addStat(statFacet); - } - } - - SearchResult searchResult = zuliaWorkPool.search(s); - - QueryResultObject result = new QueryResultObject(); - result.count = searchResult.getTotalHits(); - - List docs = new ArrayList<>(); - for (CompleteResult completeResult : searchResult.getCompleteResults()) { - Document document = completeResult.getDocument(); - docs.add(new DocumentProxyObject(document)); - } - Map> facetResults = new HashMap<>(); - - if (queryConfig.getFacets() != null) { - for (FacetConfig facet : queryConfig.getFacets()) { - List facetValues = new ArrayList<>(); - List facetCounts = searchResult.getFacetCounts(facet.getField()); - for (ZuliaQuery.FacetCount facetCount : facetCounts) { - FacetValueObject fv = new FacetValueObject(); - fv.label = facetCount.getFacet(); - fv.count = facetCount.getCount(); - facetValues.add(fv); - } - facetResults.put(facet.getField(), facetValues); - } - result.facet = new FacetProxyObject(facetResults); - } - - if (queryConfig.getStatFacets() != null) { - for (StatFacetConfig statFacet : queryConfig.getStatFacets()) { - - } - } - - result.doc = docs; - resultMap.put(queryId, result); - } - - List testResults = new ArrayList<>(); - try (Context context = Context.newBuilder("js").option("engine.WarnInterpreterOnly", "false").allowHostAccess(HostAccess.ALL).build()) { - Value jsValue = context.getBindings("js"); - for (Map.Entry countToEntry : resultMap.entrySet()) { - jsValue.putMember(countToEntry.getKey(), countToEntry.getValue()); - } - - for (Map.Entry testConfigEntry : zuliaTestConfig.getTests().entrySet()) { - TestConfig testConfig = testConfigEntry.getValue(); - - TestResult testResult = new TestResult(); - testResult.setTestId(testConfigEntry.getKey()); - testResult.setTestConfig(testConfig); - Value js = context.eval("js", testConfig.getExpr()); - testResult.setPassed(js.asBoolean()); - testResults.add(testResult); - } - - } - return testResults; - } -} diff --git a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java new file mode 100644 index 00000000..34a76725 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java @@ -0,0 +1,224 @@ +package io.zulia.testing; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import io.zulia.client.command.builder.CountFacet; +import io.zulia.client.command.builder.FilterQuery; +import io.zulia.client.command.builder.Search; +import io.zulia.client.command.builder.StatFacet; +import io.zulia.client.config.ZuliaPoolConfig; +import io.zulia.client.pool.ZuliaWorkPool; +import io.zulia.client.result.CompleteResult; +import io.zulia.client.result.SearchResult; +import io.zulia.message.ZuliaQuery; +import io.zulia.testing.config.ConnectionConfig; +import io.zulia.testing.config.FacetConfig; +import io.zulia.testing.config.IndexConfig; +import io.zulia.testing.config.QueryConfig; +import io.zulia.testing.config.StatFacetConfig; +import io.zulia.testing.config.TestConfig; +import io.zulia.testing.config.ZuliaTestConfig; +import io.zulia.testing.js.dto.DocumentProxyObject; +import io.zulia.testing.js.dto.FacetValueObject; +import io.zulia.testing.js.dto.KeyedListProxyObject; +import io.zulia.testing.js.dto.QueryResultObject; +import io.zulia.testing.js.dto.StatFacetValueObject; +import io.zulia.testing.result.TestResult; +import io.zulia.util.exception.ThrowingSupplier; +import org.bson.Document; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.HostAccess; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.proxy.ProxyObject; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ZuliaTestRunner { + private static final Logger LOG = LoggerFactory.getLogger(ZuliaTestRunner.class); + + protected final ZuliaTestConfig zuliaTestConfig; + private final Map> connectionToConnectionConfig; + + public ZuliaTestRunner(ZuliaTestConfig zuliaTestConfig) { + this.zuliaTestConfig = zuliaTestConfig; + this.connectionToConnectionConfig = buildConnectionSupplier(); + } + + public List runTests() throws Exception { + Map resultMap = buildAndRunQueries(); + return evaluateTestsWithQueryResults(resultMap); + } + + @NotNull + protected Map> buildConnectionSupplier() { + Map connectionIdToConnectionConfig = zuliaTestConfig.getConnections(); + Map indexNameToIndexConfig = zuliaTestConfig.getIndexes(); + Map> connectionToConnectionConfig = new HashMap<>(); + for (Map.Entry indexConfigEntry : connectionIdToConnectionConfig.entrySet()) { + ConnectionConfig connectionConfig = indexConfigEntry.getValue(); + + ThrowingSupplier throwingSupplier = () -> { + ZuliaPoolConfig zuliaPoolConfig = new ZuliaPoolConfig().addNode(connectionConfig.getServerAddress(), connectionConfig.getPort()); + return new ZuliaWorkPool(zuliaPoolConfig); + }; + + connectionToConnectionConfig.put(indexConfigEntry.getKey(), Suppliers.memoize(throwingSupplier)); + } + + for (Map.Entry indexConfigEntry : indexNameToIndexConfig.entrySet()) { + IndexConfig indexConfig = indexConfigEntry.getValue(); + Supplier zuliaWorkPoolSupplier = connectionToConnectionConfig.get(indexConfig.getConnection()); + if (zuliaWorkPoolSupplier == null) { + throw new IllegalArgumentException( + "Failed to find connection config <" + indexConfig.getConnection() + "> for query <" + indexConfigEntry.getKey() + ">"); + } + } + return connectionToConnectionConfig; + } + + @NotNull + protected Map buildAndRunQueries() throws Exception { + Map queries = zuliaTestConfig.getSearches(); + Map indexNameToIndexConfig = zuliaTestConfig.getIndexes(); + + Map resultMap = new HashMap<>(); + for (Map.Entry queryConfigEntry : queries.entrySet()) { + QueryConfig queryConfig = queryConfigEntry.getValue(); + IndexConfig indexConfig = indexNameToIndexConfig.get(queryConfig.getIndex()); + ZuliaWorkPool zuliaWorkPool = connectionToConnectionConfig.get(indexConfig.getConnection()).get(); + Search s = buildSearch(indexConfig.getIndexName(), queryConfig); + if (zuliaTestConfig.isLogSearches()) { + LOG.info("Running search <" + queryConfigEntry.getKey() + ">:\n" + s); + } + SearchResult searchResult = zuliaWorkPool.search(s); + QueryResultObject result = buildQueryResultObject(searchResult, queryConfig); + resultMap.put(queryConfigEntry.getKey(), result); + } + return resultMap; + } + + @NotNull + protected QueryResultObject buildQueryResultObject(SearchResult searchResult, QueryConfig queryConfig) { + QueryResultObject result = new QueryResultObject(); + result.count = searchResult.getTotalHits(); + + if (queryConfig.getAmount() > 0) { + List docs = new ArrayList<>(); + for (CompleteResult completeResult : searchResult.getCompleteResults()) { + Document document = completeResult.getDocument(); + docs.add(new DocumentProxyObject(document)); + } + result.doc = docs; + } + + if (queryConfig.getFacets() != null) { + Map> facetResults = new HashMap<>(); + for (FacetConfig facet : queryConfig.getFacets()) { + List facetValues = new ArrayList<>(); + List facetCounts = searchResult.getFacetCounts(facet.getField()); + for (ZuliaQuery.FacetCount facetCount : facetCounts) { + FacetValueObject fv = new FacetValueObject(); + fv.label = facetCount.getFacet(); + fv.count = facetCount.getCount(); + facetValues.add(fv); + } + facetResults.put(facet.getField(), facetValues); + } + result.facet = new KeyedListProxyObject<>(facetResults); + } + + if (queryConfig.getStatFacets() != null) { + Map> statFacetResults = new HashMap<>(); + for (StatFacetConfig statFacetConfig : queryConfig.getStatFacets()) { + List statFacetValuesObjects = new ArrayList<>(); + List statFacets = searchResult.getFacetFieldStat(statFacetConfig.getNumericField(), statFacetConfig.getFacetField()); + for (ZuliaQuery.FacetStats statFacet : statFacets) { + StatFacetValueObject statFacetValueObject = new StatFacetValueObject(); + statFacetValueObject.label = statFacet.getFacet(); + statFacetValueObject.docCount = statFacet.getDocCount(); + statFacetValueObject.allDocCount = statFacet.getAllDocCount(); + statFacetValueObject.valueCount = statFacet.getValueCount(); + if (statFacet.getSum().getDoubleValue() > statFacet.getSum().getLongValue()) { + statFacetValueObject.sum = statFacet.getSum().getDoubleValue(); + statFacetValueObject.max = statFacet.getMax().getDoubleValue(); + statFacetValueObject.min = statFacet.getMin().getDoubleValue(); + } + else { + statFacetValueObject.sum = statFacet.getSum().getLongValue(); + statFacetValueObject.max = statFacet.getMax().getLongValue(); + statFacetValueObject.min = statFacet.getMin().getLongValue(); + } + statFacetValuesObjects.add(statFacetValueObject); + } + statFacetResults.put(statFacetConfig.getFacetField() + "-" + statFacetConfig.getNumericField(), statFacetValuesObjects); + } + result.statFacet = new KeyedListProxyObject<>(statFacetResults); + } + + return result; + } + + @NotNull + protected Search buildSearch(String indexName, QueryConfig queryConfig) { + Search s = new Search(indexName); + s.addQuery(new FilterQuery(queryConfig.getQuery())); + s.setAmount(queryConfig.getAmount()); + + if (queryConfig.getFacets() != null) { + for (FacetConfig facet : queryConfig.getFacets()) { + CountFacet countFacet = new CountFacet(facet.getField()); + if (facet.getTopN() != 0) { + countFacet.setTopN(facet.getTopN()); + } + s.addCountFacet(countFacet); + } + } + + if (queryConfig.getStatFacets() != null) { + for (StatFacetConfig statFacetConfig : queryConfig.getStatFacets()) { + StatFacet statFacet = new StatFacet(statFacetConfig.getNumericField(), statFacetConfig.getFacetField()); + if (statFacetConfig.getTopN() != 0) { + statFacet.setTopN(statFacetConfig.getTopN()); + } + s.addStat(statFacet); + } + } + return s; + } + + @NotNull + protected List evaluateTestsWithQueryResults(Map resultMap) { + List testResults = new ArrayList<>(); + try (Context context = Context.newBuilder("js").option("engine.WarnInterpreterOnly", "false").allowHostAccess(HostAccess.ALL).build()) { + Value jsValue = context.getBindings("js"); + for (Map.Entry countToEntry : resultMap.entrySet()) { + jsValue.putMember(countToEntry.getKey(), countToEntry.getValue()); + if (zuliaTestConfig.isLogSearchResults()) { + String json = context.eval("js", "JSON.stringify(" + countToEntry.getKey() + ")").asString(); + LOG.info("Search result <" + countToEntry.getKey() + ">:\n" + json); + } + } + + for (Map.Entry testConfigEntry : zuliaTestConfig.getTests().entrySet()) { + TestConfig testConfig = testConfigEntry.getValue(); + + TestResult testResult = new TestResult(); + testResult.setTestId(testConfigEntry.getKey()); + testResult.setTestConfig(testConfig); + Value js = context.eval("js", testConfig.getExpr()); + testResult.setPassed(js.asBoolean()); + LOG.info("Test <" + testConfigEntry.getKey() + "> " + (js.asBoolean() ? "Passed" : "Failed")); + testResults.add(testResult); + } + + } + return testResults; + } + +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java index b72abb26..7d6132ef 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java @@ -6,9 +6,12 @@ public class ZuliaTestConfig { private Map connections; private Map indexes; - private Map queries; + private Map searches; private Map tests; + private boolean logSearches; + private boolean logSearchResults; + public ZuliaTestConfig() { } @@ -29,12 +32,12 @@ public void setIndexes(Map indexes) { this.indexes = indexes; } - public Map getQueries() { - return queries; + public Map getSearches() { + return searches; } - public void setQueries(Map queries) { - this.queries = queries; + public void setSearches(Map searches) { + this.searches = searches; } public Map getTests() { @@ -45,8 +48,25 @@ public void setTests(Map tests) { this.tests = tests; } + public boolean isLogSearches() { + return logSearches; + } + + public void setLogSearches(boolean logSearches) { + this.logSearches = logSearches; + } + + public boolean isLogSearchResults() { + return logSearchResults; + } + + public void setLogSearchResults(boolean logSearchResults) { + this.logSearchResults = logSearchResults; + } + @Override public String toString() { - return "DataQualityConfig{" + "connections=" + connections + ", indexes=" + indexes + ", queries=" + queries + ", tests=" + tests + '}'; + return "ZuliaTestConfig{" + "connections=" + connections + ", indexes=" + indexes + ", searches=" + searches + ", tests=" + tests + ", logSearches=" + + logSearches + ", logSearchResults=" + logSearchResults + '}'; } } diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedListProxyObject.java similarity index 79% rename from zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java rename to zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedListProxyObject.java index df5233f1..8fb728e2 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/js/dto/FacetProxyObject.java +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedListProxyObject.java @@ -7,10 +7,10 @@ import java.util.List; import java.util.Map; -public class FacetProxyObject implements ProxyObject { - private final Map> keyToFacets; +public class KeyedListProxyObject implements ProxyObject { + private final Map> keyToFacets; - public FacetProxyObject(Map> keyToFacets) { + public KeyedListProxyObject(Map> keyToFacets) { this.keyToFacets = keyToFacets; } @@ -47,7 +47,7 @@ public Object get(long index) { } @Override - public List getMember(String key) { + public List getMember(String key) { return keyToFacets.get(key); } diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java index 1d8d1303..57528370 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java @@ -14,5 +14,8 @@ public QueryResultObject() { public List doc; - public FacetProxyObject facet; + public KeyedListProxyObject facet; + + public KeyedListProxyObject statFacet; + } diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/StatFacetValueObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/StatFacetValueObject.java new file mode 100644 index 00000000..5732a7d0 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/StatFacetValueObject.java @@ -0,0 +1,11 @@ +package io.zulia.testing.js.dto; + +public class StatFacetValueObject { + public String label; + public long docCount; + public long allDocCount; + public long valueCount; + public double sum; + public double max; + public double min; +} \ No newline at end of file diff --git a/zulia-testing/src/main/resources/logback.xml b/zulia-testing/src/main/resources/logback.xml deleted file mode 100644 index e80cf380..00000000 --- a/zulia-testing/src/main/resources/logback.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - true - - - %cyan(%date{ISO8601}) %gray([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n - - - - - - - - \ No newline at end of file diff --git a/zulia-testing/src/test/resources/sample_testing.yaml b/zulia-testing/src/test/resources/sample_testing.yaml new file mode 100644 index 00000000..9426094e --- /dev/null +++ b/zulia-testing/src/test/resources/sample_testing.yaml @@ -0,0 +1,51 @@ +logSearches: true +logSearchResults: true +connections: + testConnection: + serverAddress: localhost + port: 32191 +indexes: + publications: + indexName: publications + connection: testConnection +searches: + pubsStar: + index: publications + query: "*:*" + pubsWithoutTitle: + index: publications + query: "-title:*" + pubsComplex: + index: publications + query: "title:*" + facets: + - field: pubYear + topN: 10 + - field: journalNameIso + topN: 10 + statFacets: + - facetField: pubYear + numericField: authorCount + topN: 10 + pubsWithDocument: + index: publications + query: "title:\"Understanding PCSK9 and anti-PCSK9 therapies\"" + amount: 1 +tests: + pubsCount: + expr: pubsStar.count > 30000000 + mostPubsHaveTitle: + expr: pubsWithoutTitle.count < pubsStar.count * 0.01 + oneResult: + expr: pubsWithDocument.count == 1 + resultHasCorrectAuthor: + expr: pubsWithDocument.doc[0]["authors"][0]["lastName"] == "McKenney" + topYearFacet: + expr: pubsComplex.facet["pubYear"][0].label == "2022" && pubsComplex.facet["pubYear"][0].count > 1000000 + topJournalFacet: + expr: pubsComplex.facet["journalNameIso"][0].label = "PLoS One" && pubsComplex.facet["journalNameIso"][0].count > 1000 + statFacetCheck: + expr: pubsComplex.statFacet["pubYear-authorCount"][0].label = "2022" && pubsComplex.statFacet["pubYear-authorCount"][0].docCount > 1000000 && pubsComplex.statFacet["pubYear-authorCount"][0].sum > 5000000 + + + diff --git a/zulia-tools/build.gradle.kts b/zulia-tools/build.gradle.kts new file mode 100644 index 00000000..bb404e0e --- /dev/null +++ b/zulia-tools/build.gradle.kts @@ -0,0 +1,210 @@ +plugins { + application + `java-library` +} + +description = "Zulia Tools" + +dependencies { + annotationProcessor(libs.picocli.codegen) + implementation(project(":zulia-client")) + implementation(project(":zulia-cmd-shared")) + implementation(project(":zulia-testing")) + implementation(project(":zulia-data")) + implementation(libs.snake.yaml) + implementation(libs.picocli.base) + implementation(libs.commons.compress) +} + + +val zuliaScriptTask = tasks.getByName("startScripts") +zuliaScriptTask.applicationName = "zulia" +zuliaScriptTask.mainClass.set("io.zulia.tools.cmd.Zulia") + + +val zuliaAdminScriptTask = tasks.register("createZuliaAdminScript") { + applicationName = "zuliaadmin" + mainClass.set("io.zulia.tools.cmd.ZuliaAdmin") + outputDir = zuliaScriptTask.outputDir + classpath = zuliaScriptTask.classpath +} + + +val zuliaDumpScriptTask = tasks.register("createZuliaDumpScript") { + applicationName = "zuliadump" + mainClass.set("io.zulia.tools.cmd.ZuliaDump") + outputDir = zuliaScriptTask.outputDir + classpath = zuliaScriptTask.classpath +} + +val zuliaRestoreScriptTask = tasks.register("createZuliaRestoreScript") { + applicationName = "zuliarestore" + mainClass.set("io.zulia.tools.cmd.ZuliaRestore") + outputDir = zuliaScriptTask.outputDir + classpath = zuliaScriptTask.classpath +} + +val zuliaExportScriptTask = tasks.register("createZuliaExportScript") { + applicationName = "zuliaexport" + mainClass.set("io.zulia.tools.cmd.ZuliaExport") + outputDir = zuliaScriptTask.outputDir + classpath = zuliaScriptTask.classpath +} + +val zuliaImportScriptTask = tasks.register("createZuliaImportScript") { + applicationName = "zuliaimport" + mainClass.set("io.zulia.tools.cmd.ZuliaImport") + outputDir = zuliaScriptTask.outputDir + classpath = zuliaScriptTask.classpath +} + +val zuliaTestScriptTask = tasks.register("createZuliaTestScript") { + applicationName = "zuliatest" + mainClass.set("io.zulia.tools.cmd.ZuliaTest") + outputDir = zuliaScriptTask.outputDir + classpath = zuliaScriptTask.classpath +} + +tasks.register("autocompleteDir") { + doLast { + mkdir("${layout.buildDirectory.get()}/autocomplete") + } +} + +task("picoCliZuliaAutoComplete", JavaExec::class) { + dependsOn("autocompleteDir") + mainClass.set("picocli.AutoComplete") + classpath = sourceSets["main"].runtimeClasspath + args = listOf( + "--force", + "--completionScript", + "${layout.buildDirectory.get()}/autocomplete/zulia.sh", + "io.zulia.tools.cmd.Zulia" + ) +} + + +task("picoCliZuliaAdminAutoComplete", JavaExec::class) { + dependsOn("autocompleteDir") + mainClass.set("picocli.AutoComplete") + classpath = sourceSets["main"].runtimeClasspath + args = listOf( + "--force", + "--completionScript", + "${layout.buildDirectory.get()}/autocomplete/zuliaadmin.sh", + "io.zulia.tools.cmd.ZuliaAdmin" + ) +} + +task("picoCliZuliaDumpAutoComplete", JavaExec::class) { + dependsOn("autocompleteDir") + mainClass.set("picocli.AutoComplete") + classpath = sourceSets["main"].runtimeClasspath + args = + listOf( + "--force", + "--completionScript", + "${layout.buildDirectory.get()}/autocomplete/zuliadump.sh", + "io.zulia.tools.cmd.ZuliaDump" + ) +} + + + +task("picoCliZuliaRestoreAutoComplete", JavaExec::class) { + dependsOn("autocompleteDir") + mainClass.set("picocli.AutoComplete") + classpath = sourceSets["main"].runtimeClasspath + args = listOf( + "--force", + "--completionScript", + "${layout.buildDirectory.get()}/autocomplete/zuliarestore.sh", + "io.zulia.tools.cmd.ZuliaRestore" + ) +} + + +task("picoCliZuliaImportAutoComplete", JavaExec::class) { + dependsOn("autocompleteDir") + mainClass.set("picocli.AutoComplete") + classpath = sourceSets["main"].runtimeClasspath + args = listOf( + "--force", + "--completionScript", + "${layout.buildDirectory.get()}/autocomplete/zuliaimport.sh", + "io.zulia.tools.cmd.ZuliaImport" + ) +} + +task("picoCliZuliaExportAutoComplete", JavaExec::class) { + dependsOn("autocompleteDir") + mainClass.set("picocli.AutoComplete") + classpath = sourceSets["main"].runtimeClasspath + args = listOf( + "--force", + "--completionScript", + "${layout.buildDirectory.get()}/autocomplete/zuliaexport.sh", + "io.zulia.tools.cmd.ZuliaExport" + ) +} + + +task("picoCliZuliaTestAutoComplete", JavaExec::class) { + dependsOn("autocompleteDir") + mainClass.set("picocli.AutoComplete") + classpath = sourceSets["main"].runtimeClasspath + args = listOf( + "--force", + "--completionScript", + "${layout.buildDirectory.get()}/autocomplete/zuliatest.sh", + "io.zulia.tools.cmd.ZuliaTest" + ) +} + +tasks.withType { + dependsOn( + "picoCliZuliaAutoComplete", + "picoCliZuliaAdminAutoComplete", + "picoCliZuliaDumpAutoComplete", + "picoCliZuliaRestoreAutoComplete", + "picoCliZuliaImportAutoComplete", + "picoCliZuliaExportAutoComplete", + "picoCliZuliaTestAutoComplete" + ) +} + + +distributions { + main { + contents { + from(zuliaAdminScriptTask) { + into("bin") + } + from(zuliaDumpScriptTask) { + into("bin") + } + from(zuliaRestoreScriptTask) { + into("bin") + } + from(zuliaExportScriptTask) { + into("bin") + } + from(zuliaImportScriptTask) { + into("bin") + } + from(zuliaTestScriptTask) { + into("bin") + } + from("${layout.buildDirectory.get()}/autocomplete/") { + into("bin/autocomplete") + } + + fileMode = 777 + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + } +} + + + diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/Zulia.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/Zulia.java similarity index 98% rename from zulia-server/src/main/java/io/zulia/server/cmd/Zulia.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/Zulia.java index d9a4d725..ecf68965 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/Zulia.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/Zulia.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd; +package io.zulia.tools.cmd; import io.zulia.client.command.builder.CountFacet; import io.zulia.client.command.builder.FilterQuery; @@ -12,11 +12,12 @@ import io.zulia.client.pool.ZuliaWorkPool; import io.zulia.client.result.CompleteResult; import io.zulia.client.result.SearchResult; +import io.zulia.cmd.common.ShowStackArgs; +import io.zulia.cmd.common.ZuliaCommonCmd; +import io.zulia.cmd.common.ZuliaVersionProvider; import io.zulia.message.ZuliaQuery; -import io.zulia.server.cmd.common.MultipleIndexArgs; -import io.zulia.server.cmd.common.ShowStackArgs; -import io.zulia.server.cmd.common.ZuliaVersionProvider; -import io.zulia.server.cmd.zuliaadmin.ConnectionInfo; +import io.zulia.tools.cmd.common.MultipleIndexArgs; +import io.zulia.tools.cmd.zuliaadmin.ConnectionInfo; import org.bson.Document; import picocli.CommandLine; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaAdmin.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaAdmin.java similarity index 82% rename from zulia-server/src/main/java/io/zulia/server/cmd/ZuliaAdmin.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaAdmin.java index a4a2939f..6424f0b9 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaAdmin.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaAdmin.java @@ -1,9 +1,10 @@ -package io.zulia.server.cmd; +package io.zulia.tools.cmd; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.common.ShowStackArgs; -import io.zulia.server.cmd.common.ZuliaVersionProvider; -import io.zulia.server.cmd.zuliaadmin.*; +import io.zulia.cmd.common.ShowStackArgs; +import io.zulia.cmd.common.ZuliaCommonCmd; +import io.zulia.cmd.common.ZuliaVersionProvider; +import io.zulia.tools.cmd.zuliaadmin.*; import picocli.CommandLine; @CommandLine.Command(name = "zuliaadmin", subcommands = { DisplayNodesCmd.class, DisplayIndexesCmd.class, DocCountCmd.class, ClearIndexCmd.class, diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaDump.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaDump.java similarity index 93% rename from zulia-server/src/main/java/io/zulia/server/cmd/ZuliaDump.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaDump.java index db4fc488..71861d5a 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaDump.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaDump.java @@ -1,15 +1,16 @@ -package io.zulia.server.cmd; +package io.zulia.tools.cmd; import com.google.common.base.Charsets; import com.google.protobuf.util.JsonFormat; import io.zulia.client.command.FetchLargeAssociated; import io.zulia.client.command.GetIndexConfig; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.common.MultipleIndexArgs; -import io.zulia.server.cmd.common.ShowStackArgs; -import io.zulia.server.cmd.common.ZuliaCmdUtil; -import io.zulia.server.cmd.common.ZuliaVersionProvider; -import io.zulia.server.cmd.zuliaadmin.ConnectionInfo; +import io.zulia.cmd.common.ShowStackArgs; +import io.zulia.cmd.common.ZuliaCommonCmd; +import io.zulia.cmd.common.ZuliaVersionProvider; +import io.zulia.tools.cmd.common.MultipleIndexArgs; +import io.zulia.tools.cmd.common.ZuliaCmdUtil; +import io.zulia.tools.cmd.zuliaadmin.ConnectionInfo; import io.zulia.util.pool.TaskExecutor; import io.zulia.util.pool.WorkPool; import org.slf4j.Logger; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaExport.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaExport.java similarity index 90% rename from zulia-server/src/main/java/io/zulia/server/cmd/ZuliaExport.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaExport.java index ab6fe1e7..b6ecaeec 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaExport.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaExport.java @@ -1,11 +1,12 @@ -package io.zulia.server.cmd; +package io.zulia.tools.cmd; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.common.MultipleIndexArgs; -import io.zulia.server.cmd.common.ShowStackArgs; -import io.zulia.server.cmd.common.ZuliaCmdUtil; -import io.zulia.server.cmd.common.ZuliaVersionProvider; -import io.zulia.server.cmd.zuliaadmin.ConnectionInfo; +import io.zulia.cmd.common.ShowStackArgs; +import io.zulia.cmd.common.ZuliaCommonCmd; +import io.zulia.cmd.common.ZuliaVersionProvider; +import io.zulia.tools.cmd.common.MultipleIndexArgs; +import io.zulia.tools.cmd.common.ZuliaCmdUtil; +import io.zulia.tools.cmd.zuliaadmin.ConnectionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -40,12 +41,9 @@ public class ZuliaExport implements Callable { @CommandLine.Option(names = { "-p", "--pageSize", "--rows" }, description = "Number of records in each page (default: ${DEFAULT-VALUE})") private Integer pageSize = 1000; - @Override public Integer call() throws Exception { - - ZuliaWorkPool zuliaWorkPool = connectionInfo.getConnection(); Set indexes = multipleIndexArgs.resolveIndexes(zuliaWorkPool); diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaImport.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaImport.java similarity index 85% rename from zulia-server/src/main/java/io/zulia/server/cmd/ZuliaImport.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaImport.java index 3d436360..74417a12 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaImport.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaImport.java @@ -1,12 +1,13 @@ -package io.zulia.server.cmd; +package io.zulia.tools.cmd; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.common.ShowStackArgs; -import io.zulia.server.cmd.common.SingleIndexArgs; -import io.zulia.server.cmd.common.ThreadedArgs; -import io.zulia.server.cmd.common.ZuliaCmdUtil; -import io.zulia.server.cmd.common.ZuliaVersionProvider; -import io.zulia.server.cmd.zuliaadmin.ConnectionInfo; +import io.zulia.cmd.common.ShowStackArgs; +import io.zulia.cmd.common.ZuliaCommonCmd; +import io.zulia.cmd.common.ZuliaVersionProvider; +import io.zulia.tools.cmd.common.SingleIndexArgs; +import io.zulia.tools.cmd.common.ThreadedArgs; +import io.zulia.tools.cmd.common.ZuliaCmdUtil; +import io.zulia.tools.cmd.zuliaadmin.ConnectionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaRestore.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaRestore.java similarity index 94% rename from zulia-server/src/main/java/io/zulia/server/cmd/ZuliaRestore.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaRestore.java index 804e7082..84bf7a39 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/ZuliaRestore.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaRestore.java @@ -1,15 +1,16 @@ -package io.zulia.server.cmd; +package io.zulia.tools.cmd; import com.google.common.base.Charsets; import com.google.protobuf.util.JsonFormat; import io.zulia.client.config.ClientIndexConfig; import io.zulia.client.pool.ZuliaWorkPool; +import io.zulia.cmd.common.ShowStackArgs; +import io.zulia.cmd.common.ZuliaCommonCmd; +import io.zulia.cmd.common.ZuliaVersionProvider; import io.zulia.message.ZuliaIndex; -import io.zulia.server.cmd.common.ShowStackArgs; -import io.zulia.server.cmd.common.ThreadedArgs; -import io.zulia.server.cmd.common.ZuliaCmdUtil; -import io.zulia.server.cmd.common.ZuliaVersionProvider; -import io.zulia.server.cmd.zuliaadmin.ConnectionInfo; +import io.zulia.tools.cmd.common.ThreadedArgs; +import io.zulia.tools.cmd.common.ZuliaCmdUtil; +import io.zulia.tools.cmd.zuliaadmin.ConnectionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; diff --git a/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java new file mode 100644 index 00000000..2276edeb --- /dev/null +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java @@ -0,0 +1,68 @@ +package io.zulia.tools.cmd; + +import io.zulia.cmd.common.ZuliaCommonCmd; +import io.zulia.cmd.common.ZuliaVersionProvider; +import io.zulia.data.output.FileDataOutputStream; +import io.zulia.data.target.spreadsheet.csv.CSVDataTarget; +import io.zulia.testing.ZuliaTestRunner; +import io.zulia.testing.config.ZuliaTestConfig; +import io.zulia.testing.result.TestResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; +import picocli.CommandLine; + +import java.io.FileInputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.concurrent.Callable; + +@CommandLine.Command(name = "zuliatest", versionProvider = ZuliaVersionProvider.class, scope = CommandLine.ScopeType.INHERIT) +public class ZuliaTest implements Callable { + + private static final Logger LOG = LoggerFactory.getLogger(ZuliaTest.class); + + @CommandLine.Option(names = "--testConfig", description = "Full path to the test config yaml file", required = true) + private String testConfig; + + @CommandLine.Option(names = "--testOutput", description = "Full path to the test output csv file", required = true) + private String testOutput; + + @Override + public Integer call() throws Exception { + + Path testConfigPath = Paths.get(testConfig); + if (Files.exists(testConfigPath)) { + Yaml yaml = new Yaml(); + try (FileInputStream fileInputStream = new FileInputStream(testConfigPath.toFile())) { + ZuliaTestConfig zuliaTestConfig = yaml.loadAs(fileInputStream, ZuliaTestConfig.class); + LOG.info("Running tests from <" + testConfig + ">"); + ZuliaTestRunner zuliaTestRunner = new ZuliaTestRunner(zuliaTestConfig); + List testResults = zuliaTestRunner.runTests(); + + LOG.info("Writing results to <" + testOutput + ">"); + FileDataOutputStream dataOutputStream = FileDataOutputStream.from(testOutput, true); + try (CSVDataTarget csvDataTarget = CSVDataTarget.withDefaults(dataOutputStream)) { + for (TestResult testResult : testResults) { + csvDataTarget.writeRow(testResult.getTestId(), testResult.isPassed() ? "PASS" : "FAIL"); + } + } + + } + + } + else { + System.err.println("Yaml configuration file <" + testConfig + "> does not exist or is not readable"); + System.exit(9); + } + + return CommandLine.ExitCode.OK; + } + + public static void main(String[] args) { + ZuliaCommonCmd.runCommandLine(new ZuliaTest(), args); + } + +} diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/common/AliasArgs.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/AliasArgs.java similarity index 97% rename from zulia-server/src/main/java/io/zulia/server/cmd/common/AliasArgs.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/common/AliasArgs.java index ba7a3a09..cfb57004 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/common/AliasArgs.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/AliasArgs.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd.common; +package io.zulia.tools.cmd.common; import io.zulia.client.pool.ZuliaWorkPool; import io.zulia.message.ZuliaIndex.IndexAlias; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/common/MultipleIndexArgs.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/MultipleIndexArgs.java similarity index 96% rename from zulia-server/src/main/java/io/zulia/server/cmd/common/MultipleIndexArgs.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/common/MultipleIndexArgs.java index 55ea5a18..e7a15510 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/common/MultipleIndexArgs.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/MultipleIndexArgs.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd.common; +package io.zulia.tools.cmd.common; import io.zulia.client.pool.ZuliaWorkPool; import picocli.CommandLine; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/common/SingleIndexArgs.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/SingleIndexArgs.java similarity index 86% rename from zulia-server/src/main/java/io/zulia/server/cmd/common/SingleIndexArgs.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/common/SingleIndexArgs.java index b413b3c8..86eed283 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/common/SingleIndexArgs.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/SingleIndexArgs.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd.common; +package io.zulia.tools.cmd.common; import picocli.CommandLine; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/common/ThreadedArgs.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/ThreadedArgs.java similarity index 88% rename from zulia-server/src/main/java/io/zulia/server/cmd/common/ThreadedArgs.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/common/ThreadedArgs.java index 3e9d6cc7..96342b87 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/common/ThreadedArgs.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/ThreadedArgs.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd.common; +package io.zulia.tools.cmd.common; import picocli.CommandLine; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/common/ZuliaCmdUtil.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/ZuliaCmdUtil.java similarity index 99% rename from zulia-server/src/main/java/io/zulia/server/cmd/common/ZuliaCmdUtil.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/common/ZuliaCmdUtil.java index 1c36ba2c..bbd13655 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/common/ZuliaCmdUtil.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/common/ZuliaCmdUtil.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd.common; +package io.zulia.tools.cmd.common; import com.google.common.base.Charsets; import io.zulia.ZuliaConstants; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ClearIndexCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ClearIndexCmd.java similarity index 84% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ClearIndexCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ClearIndexCmd.java index b1f2a637..e22796c4 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ClearIndexCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ClearIndexCmd.java @@ -1,8 +1,8 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.common.MultipleIndexArgs; +import io.zulia.tools.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.common.MultipleIndexArgs; import picocli.CommandLine; import java.util.Set; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ConnectionInfo.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ConnectionInfo.java similarity index 96% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ConnectionInfo.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ConnectionInfo.java index 4eab4ad7..70fe0783 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ConnectionInfo.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ConnectionInfo.java @@ -1,4 +1,4 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.config.ZuliaPoolConfig; import io.zulia.client.pool.ZuliaWorkPool; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/CreateAliasCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/CreateAliasCmd.java similarity index 91% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/CreateAliasCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/CreateAliasCmd.java index b40a876f..fd2e4444 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/CreateAliasCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/CreateAliasCmd.java @@ -1,7 +1,7 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.ZuliaAdmin; import picocli.CommandLine; import java.util.concurrent.Callable; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DeleteAliasCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DeleteAliasCmd.java similarity index 85% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DeleteAliasCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DeleteAliasCmd.java index 0be15fb1..717283ea 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DeleteAliasCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DeleteAliasCmd.java @@ -1,8 +1,8 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.common.AliasArgs; +import io.zulia.tools.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.common.AliasArgs; import picocli.CommandLine; import java.util.Set; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DeleteIndexCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DeleteIndexCmd.java similarity index 87% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DeleteIndexCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DeleteIndexCmd.java index 97247064..ea048da3 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DeleteIndexCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DeleteIndexCmd.java @@ -1,8 +1,8 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.common.MultipleIndexArgs; +import io.zulia.tools.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.common.MultipleIndexArgs; import picocli.CommandLine; import java.util.Set; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DisplayAliasesCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DisplayAliasesCmd.java similarity index 87% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DisplayAliasesCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DisplayAliasesCmd.java index 5509111f..10092aa9 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DisplayAliasesCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DisplayAliasesCmd.java @@ -1,9 +1,9 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; +import io.zulia.cmd.common.ZuliaCommonCmd; import io.zulia.message.ZuliaIndex; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.ZuliaCommonCmd; +import io.zulia.tools.cmd.ZuliaAdmin; import picocli.CommandLine; import java.util.List; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DisplayIndexesCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DisplayIndexesCmd.java similarity index 94% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DisplayIndexesCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DisplayIndexesCmd.java index a8aa1687..f51a8e65 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DisplayIndexesCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DisplayIndexesCmd.java @@ -1,11 +1,11 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; import io.zulia.client.result.GetIndexesResult; +import io.zulia.cmd.common.ZuliaCommonCmd; import io.zulia.message.ZuliaBase; import io.zulia.message.ZuliaIndex; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.ZuliaCommonCmd; +import io.zulia.tools.cmd.ZuliaAdmin; import picocli.CommandLine; import java.util.HashMap; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DisplayNodesCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DisplayNodesCmd.java similarity index 92% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DisplayNodesCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DisplayNodesCmd.java index fd183f99..0dcfc0a7 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DisplayNodesCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DisplayNodesCmd.java @@ -1,10 +1,10 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; import io.zulia.client.result.GetNodesResult; +import io.zulia.cmd.common.ZuliaCommonCmd; import io.zulia.message.ZuliaBase; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.ZuliaCommonCmd; +import io.zulia.tools.cmd.ZuliaAdmin; import picocli.CommandLine; import java.time.Instant; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DocCountCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DocCountCmd.java similarity index 92% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DocCountCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DocCountCmd.java index 55dc35a9..b62d1da5 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/DocCountCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/DocCountCmd.java @@ -1,10 +1,10 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; import io.zulia.client.result.GetNumberOfDocsResult; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.ZuliaCommonCmd; -import io.zulia.server.cmd.common.MultipleIndexArgs; +import io.zulia.cmd.common.ZuliaCommonCmd; +import io.zulia.tools.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.common.MultipleIndexArgs; import picocli.CommandLine; import java.util.ArrayList; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ExportAliasesCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ExportAliasesCmd.java similarity index 91% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ExportAliasesCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ExportAliasesCmd.java index ecc42259..abbcc372 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ExportAliasesCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ExportAliasesCmd.java @@ -1,11 +1,11 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import com.google.common.base.Charsets; import com.google.protobuf.util.JsonFormat; import io.zulia.client.pool.ZuliaWorkPool; import io.zulia.message.ZuliaIndex; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.common.AliasArgs; +import io.zulia.tools.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.common.AliasArgs; import picocli.CommandLine; import java.io.File; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/FetchAssociatedFileCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/FetchAssociatedFileCmd.java similarity index 90% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/FetchAssociatedFileCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/FetchAssociatedFileCmd.java index 2b221192..eafc44c1 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/FetchAssociatedFileCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/FetchAssociatedFileCmd.java @@ -1,9 +1,9 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.command.FetchLargeAssociated; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.common.SingleIndexArgs; +import io.zulia.tools.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.common.SingleIndexArgs; import picocli.CommandLine; import java.io.File; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ImportAliasesCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ImportAliasesCmd.java similarity index 93% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ImportAliasesCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ImportAliasesCmd.java index 38c74c3f..db9dfb02 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ImportAliasesCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ImportAliasesCmd.java @@ -1,10 +1,10 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import com.google.protobuf.util.JsonFormat; import io.zulia.client.command.CreateIndexAlias; import io.zulia.client.pool.ZuliaWorkPool; import io.zulia.message.ZuliaIndex; -import io.zulia.server.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.ZuliaAdmin; import picocli.CommandLine; import java.io.File; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/OptimizeIndexCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/OptimizeIndexCmd.java similarity index 84% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/OptimizeIndexCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/OptimizeIndexCmd.java index 1474b31e..402cf623 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/OptimizeIndexCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/OptimizeIndexCmd.java @@ -1,8 +1,8 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.common.MultipleIndexArgs; +import io.zulia.tools.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.common.MultipleIndexArgs; import picocli.CommandLine; import java.util.Set; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ReindexCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ReindexCmd.java similarity index 85% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ReindexCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ReindexCmd.java index 47f7c1f1..db7399e5 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/ReindexCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/ReindexCmd.java @@ -1,8 +1,8 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.common.MultipleIndexArgs; +import io.zulia.tools.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.common.MultipleIndexArgs; import picocli.CommandLine; import java.util.Set; diff --git a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/StoreAssociatedFileCmd.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/StoreAssociatedFileCmd.java similarity index 91% rename from zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/StoreAssociatedFileCmd.java rename to zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/StoreAssociatedFileCmd.java index 85e701f8..5cc339f8 100644 --- a/zulia-server/src/main/java/io/zulia/server/cmd/zuliaadmin/StoreAssociatedFileCmd.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/zuliaadmin/StoreAssociatedFileCmd.java @@ -1,9 +1,9 @@ -package io.zulia.server.cmd.zuliaadmin; +package io.zulia.tools.cmd.zuliaadmin; import io.zulia.client.command.StoreLargeAssociated; import io.zulia.client.pool.ZuliaWorkPool; -import io.zulia.server.cmd.ZuliaAdmin; -import io.zulia.server.cmd.common.SingleIndexArgs; +import io.zulia.tools.cmd.ZuliaAdmin; +import io.zulia.tools.cmd.common.SingleIndexArgs; import picocli.CommandLine; import java.io.File; diff --git a/zulia-tools/src/test/resources/sample_testing.yaml b/zulia-tools/src/test/resources/sample_testing.yaml new file mode 100644 index 00000000..cee9625e --- /dev/null +++ b/zulia-tools/src/test/resources/sample_testing.yaml @@ -0,0 +1,42 @@ +connections: + testConnection: + serverAddress: 192.168.1.30 + port: 32191 +indexes: + publications: + indexName: publications + connection: testConnection +queries: + pubsStar: + index: publications + query: "*:*" + pubsWithoutTitle: + index: publications + query: "-title:*" + pubsTest: + index: publications + query: "title:*" + amount: 1 + facets: + - field: pubYear + topN: 10 + - field: journalName + topN: 10 + statFacets: + - facetField: pubYear + numericField: rcr + topN: 10 +tests: + pubsCount: + expr: pubsStar.count > 200000 + pubsWithTitleCount: + expr: pubsWithoutTitle.count < pubsStar.count * 0.01 + test1: + expr: pubsTest.count > 1 + test2: + expr: pubsTest.doc[0]["authors"][0]["lastName"] == "Abumere" + test3: + expr: pubsTest.facet["pubYear"][0].label == "2022" && pubsTest.facet["pubYear"][0].count > 1000000 + + + From 6e18aa8e1fc4216d73e18008a14e591281735632 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Sun, 7 Apr 2024 11:23:00 -0400 Subject: [PATCH 06/12] add numeric stat to testable queries refactor to create KeyedProxyObject and KeyedListProxyObject --- .../io/zulia/testing/ZuliaTestRunner.java | 120 ++++++++++++------ .../zulia/testing/config/NumStatConfig.java | 45 +++++++ .../io/zulia/testing/config/QueryConfig.java | 12 +- .../testing/js/dto/KeyedListProxyObject.java | 46 +------ .../testing/js/dto/KeyedProxyObject.java | 53 ++++++++ .../testing/js/dto/PercentileValueObject.java | 6 + .../testing/js/dto/QueryResultObject.java | 2 + .../testing/js/dto/StatFacetValueObject.java | 3 + .../src/test/resources/sample_testing.yaml | 14 +- .../java/io/zulia/tools/cmd/ZuliaTest.java | 4 + 10 files changed, 219 insertions(+), 86 deletions(-) create mode 100644 zulia-testing/src/main/java/io/zulia/testing/config/NumStatConfig.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedProxyObject.java create mode 100644 zulia-testing/src/main/java/io/zulia/testing/js/dto/PercentileValueObject.java diff --git a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java index 34a76725..fc717be7 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java +++ b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java @@ -4,6 +4,7 @@ import com.google.common.base.Suppliers; import io.zulia.client.command.builder.CountFacet; import io.zulia.client.command.builder.FilterQuery; +import io.zulia.client.command.builder.NumericStat; import io.zulia.client.command.builder.Search; import io.zulia.client.command.builder.StatFacet; import io.zulia.client.config.ZuliaPoolConfig; @@ -14,6 +15,7 @@ import io.zulia.testing.config.ConnectionConfig; import io.zulia.testing.config.FacetConfig; import io.zulia.testing.config.IndexConfig; +import io.zulia.testing.config.NumStatConfig; import io.zulia.testing.config.QueryConfig; import io.zulia.testing.config.StatFacetConfig; import io.zulia.testing.config.TestConfig; @@ -21,6 +23,8 @@ import io.zulia.testing.js.dto.DocumentProxyObject; import io.zulia.testing.js.dto.FacetValueObject; import io.zulia.testing.js.dto.KeyedListProxyObject; +import io.zulia.testing.js.dto.KeyedProxyObject; +import io.zulia.testing.js.dto.PercentileValueObject; import io.zulia.testing.js.dto.QueryResultObject; import io.zulia.testing.js.dto.StatFacetValueObject; import io.zulia.testing.result.TestResult; @@ -103,6 +107,47 @@ protected Map buildAndRunQueries() throws Exception { return resultMap; } + @NotNull + protected Search buildSearch(String indexName, QueryConfig queryConfig) { + Search s = new Search(indexName); + s.addQuery(new FilterQuery(queryConfig.getQuery())); + s.setAmount(queryConfig.getAmount()); + + if (queryConfig.getFacets() != null) { + for (FacetConfig facet : queryConfig.getFacets()) { + CountFacet countFacet = new CountFacet(facet.getField()); + if (facet.getTopN() != 0) { + countFacet.setTopN(facet.getTopN()); + } + s.addCountFacet(countFacet); + } + } + + if (queryConfig.getStatFacets() != null) { + for (StatFacetConfig statFacetConfig : queryConfig.getStatFacets()) { + StatFacet statFacet = new StatFacet(statFacetConfig.getNumericField(), statFacetConfig.getFacetField()); + if (statFacetConfig.getTopN() != 0) { + statFacet.setTopN(statFacetConfig.getTopN()); + } + s.addStat(statFacet); + } + } + if (queryConfig.getNumStats() != null) { + for (NumStatConfig numStatConfig : queryConfig.getNumStats()) { + NumericStat numericStat = new NumericStat(numStatConfig.getNumericField()); + if (numStatConfig.getPercentilePrecision() > 0) { + numericStat.setPercentilePrecision(numStatConfig.getPercentilePrecision()); + } + if (numStatConfig.getPercentiles() != null) { + numericStat.setPercentiles(numStatConfig.getPercentiles()); + } + s.addStat(numericStat); + } + } + + return s; + } + @NotNull protected QueryResultObject buildQueryResultObject(SearchResult searchResult, QueryConfig queryConfig) { QueryResultObject result = new QueryResultObject(); @@ -139,21 +184,7 @@ protected QueryResultObject buildQueryResultObject(SearchResult searchResult, Qu List statFacetValuesObjects = new ArrayList<>(); List statFacets = searchResult.getFacetFieldStat(statFacetConfig.getNumericField(), statFacetConfig.getFacetField()); for (ZuliaQuery.FacetStats statFacet : statFacets) { - StatFacetValueObject statFacetValueObject = new StatFacetValueObject(); - statFacetValueObject.label = statFacet.getFacet(); - statFacetValueObject.docCount = statFacet.getDocCount(); - statFacetValueObject.allDocCount = statFacet.getAllDocCount(); - statFacetValueObject.valueCount = statFacet.getValueCount(); - if (statFacet.getSum().getDoubleValue() > statFacet.getSum().getLongValue()) { - statFacetValueObject.sum = statFacet.getSum().getDoubleValue(); - statFacetValueObject.max = statFacet.getMax().getDoubleValue(); - statFacetValueObject.min = statFacet.getMin().getDoubleValue(); - } - else { - statFacetValueObject.sum = statFacet.getSum().getLongValue(); - statFacetValueObject.max = statFacet.getMax().getLongValue(); - statFacetValueObject.min = statFacet.getMin().getLongValue(); - } + StatFacetValueObject statFacetValueObject = getStatFacetValueObject(statFacet); statFacetValuesObjects.add(statFacetValueObject); } statFacetResults.put(statFacetConfig.getFacetField() + "-" + statFacetConfig.getNumericField(), statFacetValuesObjects); @@ -161,37 +192,49 @@ protected QueryResultObject buildQueryResultObject(SearchResult searchResult, Qu result.statFacet = new KeyedListProxyObject<>(statFacetResults); } + if (queryConfig.getNumStats() != null) { + Map numStatResults = new HashMap<>(); + for (NumStatConfig numStatConfig : queryConfig.getNumStats()) { + ZuliaQuery.FacetStats numericFieldStat = searchResult.getNumericFieldStat(numStatConfig.getNumericField()); + StatFacetValueObject statFacetValueObject = getStatFacetValueObject(numericFieldStat); + numStatResults.put(numStatConfig.getNumericField(), statFacetValueObject); + } + result.numStat = new KeyedProxyObject<>(numStatResults); + } + return result; } - @NotNull - protected Search buildSearch(String indexName, QueryConfig queryConfig) { - Search s = new Search(indexName); - s.addQuery(new FilterQuery(queryConfig.getQuery())); - s.setAmount(queryConfig.getAmount()); - - if (queryConfig.getFacets() != null) { - for (FacetConfig facet : queryConfig.getFacets()) { - CountFacet countFacet = new CountFacet(facet.getField()); - if (facet.getTopN() != 0) { - countFacet.setTopN(facet.getTopN()); - } - s.addCountFacet(countFacet); - } + private static @NotNull StatFacetValueObject getStatFacetValueObject(ZuliaQuery.FacetStats statFacet) { + StatFacetValueObject statFacetValueObject = new StatFacetValueObject(); + statFacetValueObject.label = statFacet.getFacet(); + statFacetValueObject.docCount = statFacet.getDocCount(); + statFacetValueObject.allDocCount = statFacet.getAllDocCount(); + statFacetValueObject.valueCount = statFacet.getValueCount(); + if (statFacet.getSum().getDoubleValue() > statFacet.getSum().getLongValue()) { + statFacetValueObject.sum = statFacet.getSum().getDoubleValue(); + statFacetValueObject.max = statFacet.getMax().getDoubleValue(); + statFacetValueObject.min = statFacet.getMin().getDoubleValue(); } - - if (queryConfig.getStatFacets() != null) { - for (StatFacetConfig statFacetConfig : queryConfig.getStatFacets()) { - StatFacet statFacet = new StatFacet(statFacetConfig.getNumericField(), statFacetConfig.getFacetField()); - if (statFacetConfig.getTopN() != 0) { - statFacet.setTopN(statFacetConfig.getTopN()); - } - s.addStat(statFacet); + else { + statFacetValueObject.sum = statFacet.getSum().getLongValue(); + statFacetValueObject.max = statFacet.getMax().getLongValue(); + statFacetValueObject.min = statFacet.getMin().getLongValue(); + } + if (!statFacet.getPercentilesList().isEmpty()) { + List percentileValueObjects = new ArrayList<>(); + for (ZuliaQuery.Percentile percentile : statFacet.getPercentilesList()) { + PercentileValueObject percentileValueObject = new PercentileValueObject(); + percentileValueObject.point = percentile.getPoint(); + percentileValueObject.value = percentile.getValue(); + percentileValueObjects.add(percentileValueObject); } + statFacetValueObject.percentiles = percentileValueObjects; } - return s; + return statFacetValueObject; } + @NotNull protected List evaluateTestsWithQueryResults(Map resultMap) { List testResults = new ArrayList<>(); @@ -206,6 +249,7 @@ protected List evaluateTestsWithQueryResults(Map testConfigEntry : zuliaTestConfig.getTests().entrySet()) { + LOG.info("Running Test <" + testConfigEntry.getKey() + ">"); TestConfig testConfig = testConfigEntry.getValue(); TestResult testResult = new TestResult(); diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/NumStatConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/NumStatConfig.java new file mode 100644 index 00000000..f17c5782 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/config/NumStatConfig.java @@ -0,0 +1,45 @@ +package io.zulia.testing.config; + +import java.util.List; + +public class NumStatConfig { + + private String numericField; + + private double percentilePrecision; + + private List percentiles; + + public NumStatConfig() { + } + + public String getNumericField() { + return numericField; + } + + public void setNumericField(String numericField) { + this.numericField = numericField; + } + + public double getPercentilePrecision() { + return percentilePrecision; + } + + public void setPercentilePrecision(double percentilePrecision) { + this.percentilePrecision = percentilePrecision; + } + + public List getPercentiles() { + return percentiles; + } + + public void setPercentiles(List percentiles) { + this.percentiles = percentiles; + } + + @Override + public String toString() { + return "NumStatConfig{" + "numericField='" + numericField + '\'' + ", percentilePrecision=" + percentilePrecision + ", percentiles=" + percentiles + + '}'; + } +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java index 23f0a2f3..52bd45a8 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java @@ -12,6 +12,8 @@ public final class QueryConfig { private List statFacets; + private List numStats; + public QueryConfig() { } @@ -56,9 +58,17 @@ public void setStatFacets(List statFacets) { this.statFacets = statFacets; } + public List getNumStats() { + return numStats; + } + + public void setNumStats(List numStats) { + this.numStats = numStats; + } + @Override public String toString() { return "QueryConfig{" + "index='" + index + '\'' + ", query='" + query + '\'' + ", amount=" + amount + ", facets=" + facets + ", statFacets=" - + statFacets + '}'; + + statFacets + ", numStats=" + numStats + '}'; } } diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedListProxyObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedListProxyObject.java index 8fb728e2..49f9b54e 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedListProxyObject.java +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedListProxyObject.java @@ -1,54 +1,12 @@ package io.zulia.testing.js.dto; -import org.graalvm.polyglot.Value; -import org.graalvm.polyglot.proxy.ProxyArray; -import org.graalvm.polyglot.proxy.ProxyObject; - import java.util.List; import java.util.Map; -public class KeyedListProxyObject implements ProxyObject { - private final Map> keyToFacets; +public class KeyedListProxyObject extends KeyedProxyObject> { public KeyedListProxyObject(Map> keyToFacets) { - this.keyToFacets = keyToFacets; - } - - @Override - public void putMember(String key, Value value) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean hasMember(String key) { - return keyToFacets.containsKey(key); - } - - @Override - public Object getMemberKeys() { - return new ProxyArray() { - private final Object[] keys = keyToFacets.keySet().toArray(); - - public void set(long index, Value value) { - throw new UnsupportedOperationException(); - } - - public long getSize() { - return keys.length; - } - - public Object get(long index) { - if (index < 0 || index > Integer.MAX_VALUE) { - throw new ArrayIndexOutOfBoundsException(); - } - return keys[(int) index]; - } - }; - } - - @Override - public List getMember(String key) { - return keyToFacets.get(key); + super(keyToFacets); } } \ No newline at end of file diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedProxyObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedProxyObject.java new file mode 100644 index 00000000..2192f991 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/KeyedProxyObject.java @@ -0,0 +1,53 @@ +package io.zulia.testing.js.dto; + +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.proxy.ProxyArray; +import org.graalvm.polyglot.proxy.ProxyObject; + +import java.util.Map; + +public class KeyedProxyObject implements ProxyObject { + private final Map keyToFacets; + + public KeyedProxyObject(Map keyToFacets) { + this.keyToFacets = keyToFacets; + } + + @Override + public void putMember(String key, Value value) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasMember(String key) { + return keyToFacets.containsKey(key); + } + + @Override + public Object getMemberKeys() { + return new ProxyArray() { + private final Object[] keys = keyToFacets.keySet().toArray(); + + public void set(long index, Value value) { + throw new UnsupportedOperationException(); + } + + public long getSize() { + return keys.length; + } + + public Object get(long index) { + if (index < 0 || index > Integer.MAX_VALUE) { + throw new ArrayIndexOutOfBoundsException(); + } + return keys[(int) index]; + } + }; + } + + @Override + public T getMember(String key) { + return keyToFacets.get(key); + } + +} \ No newline at end of file diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/PercentileValueObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/PercentileValueObject.java new file mode 100644 index 00000000..66c14532 --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/PercentileValueObject.java @@ -0,0 +1,6 @@ +package io.zulia.testing.js.dto; + +public class PercentileValueObject { + public double point; + public double value; +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java index 57528370..eac716a0 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/QueryResultObject.java @@ -18,4 +18,6 @@ public QueryResultObject() { public KeyedListProxyObject statFacet; + public KeyedProxyObject numStat; + } diff --git a/zulia-testing/src/main/java/io/zulia/testing/js/dto/StatFacetValueObject.java b/zulia-testing/src/main/java/io/zulia/testing/js/dto/StatFacetValueObject.java index 5732a7d0..d893fb87 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/js/dto/StatFacetValueObject.java +++ b/zulia-testing/src/main/java/io/zulia/testing/js/dto/StatFacetValueObject.java @@ -1,5 +1,7 @@ package io.zulia.testing.js.dto; +import java.util.List; + public class StatFacetValueObject { public String label; public long docCount; @@ -8,4 +10,5 @@ public class StatFacetValueObject { public double sum; public double max; public double min; + public List percentiles; } \ No newline at end of file diff --git a/zulia-testing/src/test/resources/sample_testing.yaml b/zulia-testing/src/test/resources/sample_testing.yaml index 9426094e..f522d2e6 100644 --- a/zulia-testing/src/test/resources/sample_testing.yaml +++ b/zulia-testing/src/test/resources/sample_testing.yaml @@ -27,6 +27,12 @@ searches: - facetField: pubYear numericField: authorCount topN: 10 + pubsComplex2: + index: publications + query: "title:cancer" + numStats: + - numericField: pubYear + percentiles: [ 0.1, 0.5, 0.9 ] pubsWithDocument: index: publications query: "title:\"Understanding PCSK9 and anti-PCSK9 therapies\"" @@ -45,7 +51,9 @@ tests: topJournalFacet: expr: pubsComplex.facet["journalNameIso"][0].label = "PLoS One" && pubsComplex.facet["journalNameIso"][0].count > 1000 statFacetCheck: - expr: pubsComplex.statFacet["pubYear-authorCount"][0].label = "2022" && pubsComplex.statFacet["pubYear-authorCount"][0].docCount > 1000000 && pubsComplex.statFacet["pubYear-authorCount"][0].sum > 5000000 - - + expr: pubsComplex.statFacet["pubYear-authorCount"][0].label = "2022" && pubsComplex.statFacet["pubYear-authorCount"][0].docCount > 1000000 && pubsComplex.statFacet["pubYear-authorCount"][0].sum > 5000000 + numStatCheck: + expr: (pubsComplex2.numStat["pubYear"].sum/pubsComplex2.numStat["pubYear"].docCount) > 2008 + numStatCheck2: + expr: (pubsComplex2.numStat["pubYear"].percentiles[0].value < 1990) && (pubsComplex2.numStat["pubYear"].percentiles[2].value > 2021) diff --git a/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java index 2276edeb..9302f4a9 100644 --- a/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java @@ -1,5 +1,6 @@ package io.zulia.tools.cmd; +import io.zulia.cmd.common.ShowStackArgs; import io.zulia.cmd.common.ZuliaCommonCmd; import io.zulia.cmd.common.ZuliaVersionProvider; import io.zulia.data.output.FileDataOutputStream; @@ -30,6 +31,9 @@ public class ZuliaTest implements Callable { @CommandLine.Option(names = "--testOutput", description = "Full path to the test output csv file", required = true) private String testOutput; + @CommandLine.Mixin + private ShowStackArgs showStackArgs; + @Override public Integer call() throws Exception { From 807c3fd4ddef5aa7e3930f72588c659ab8b0f3b6 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Sun, 7 Apr 2024 13:36:16 -0400 Subject: [PATCH 07/12] ensure task exits with failure if test fails add more complete query support add support for document fields --- .../io/zulia/testing/ZuliaTestRunner.java | 56 +++++++++++-- .../io/zulia/testing/config/QueryConfig.java | 69 ++++++--------- .../io/zulia/testing/config/SearchConfig.java | 84 +++++++++++++++++++ .../zulia/testing/config/ZuliaTestConfig.java | 6 +- .../java/io/zulia/tools/cmd/ZuliaTest.java | 8 ++ 5 files changed, 168 insertions(+), 55 deletions(-) create mode 100644 zulia-testing/src/main/java/io/zulia/testing/config/SearchConfig.java diff --git a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java index fc717be7..4962ae08 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java +++ b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java @@ -5,6 +5,7 @@ import io.zulia.client.command.builder.CountFacet; import io.zulia.client.command.builder.FilterQuery; import io.zulia.client.command.builder.NumericStat; +import io.zulia.client.command.builder.ScoredQuery; import io.zulia.client.command.builder.Search; import io.zulia.client.command.builder.StatFacet; import io.zulia.client.config.ZuliaPoolConfig; @@ -17,6 +18,7 @@ import io.zulia.testing.config.IndexConfig; import io.zulia.testing.config.NumStatConfig; import io.zulia.testing.config.QueryConfig; +import io.zulia.testing.config.SearchConfig; import io.zulia.testing.config.StatFacetConfig; import io.zulia.testing.config.TestConfig; import io.zulia.testing.config.ZuliaTestConfig; @@ -43,6 +45,11 @@ import java.util.List; import java.util.Map; +import static io.zulia.message.ZuliaQuery.Query.QueryType.FILTER; +import static io.zulia.message.ZuliaQuery.Query.QueryType.FILTER_NOT; +import static io.zulia.message.ZuliaQuery.Query.QueryType.SCORE_MUST; +import static io.zulia.message.ZuliaQuery.Query.QueryType.SCORE_SHOULD; + public class ZuliaTestRunner { private static final Logger LOG = LoggerFactory.getLogger(ZuliaTestRunner.class); @@ -88,12 +95,12 @@ protected Map> buildConnectionSupplier() { @NotNull protected Map buildAndRunQueries() throws Exception { - Map queries = zuliaTestConfig.getSearches(); + Map searches = zuliaTestConfig.getSearches(); Map indexNameToIndexConfig = zuliaTestConfig.getIndexes(); Map resultMap = new HashMap<>(); - for (Map.Entry queryConfigEntry : queries.entrySet()) { - QueryConfig queryConfig = queryConfigEntry.getValue(); + for (Map.Entry queryConfigEntry : searches.entrySet()) { + SearchConfig queryConfig = queryConfigEntry.getValue(); IndexConfig indexConfig = indexNameToIndexConfig.get(queryConfig.getIndex()); ZuliaWorkPool zuliaWorkPool = connectionToConnectionConfig.get(indexConfig.getConnection()).get(); Search s = buildSearch(indexConfig.getIndexName(), queryConfig); @@ -108,9 +115,45 @@ protected Map buildAndRunQueries() throws Exception { } @NotNull - protected Search buildSearch(String indexName, QueryConfig queryConfig) { + protected Search buildSearch(String indexName, SearchConfig queryConfig) { Search s = new Search(indexName); - s.addQuery(new FilterQuery(queryConfig.getQuery())); + List queries = queryConfig.getQueries(); + if (queries != null) { + for (QueryConfig query : queries) { + if (FILTER == query.getQueryType() || FILTER_NOT == query.getQueryType()) { + FilterQuery queryBuilder = new FilterQuery(query.getQ()); + if (FILTER_NOT == query.getQueryType()) { + queryBuilder.exclude(); + } + if (query.getQf() != null) { + queryBuilder.addQueryFields(query.getQf()); + } + if (query.getMm() > 0) { + queryBuilder.setMinShouldMatch(query.getMm()); + } + s.addQuery(queryBuilder); + } + else if (SCORE_MUST == query.getQueryType() || SCORE_SHOULD == query.getQueryType()) { + ScoredQuery queryBuilder = new ScoredQuery(query.getQ()); + queryBuilder.setMust((SCORE_MUST == query.getQueryType())); + if (query.getQf() != null) { + queryBuilder.addQueryFields(query.getQf()); + } + if (query.getMm() > 0) { + queryBuilder.setMinShouldMatch(query.getMm()); + } + s.addQuery(queryBuilder); + } + else { + throw new IllegalArgumentException("Unsupported query type <" + query.getQueryType() + ">"); + } + } + } + + if (queryConfig.getDocumentFields() != null) { + s.addDocumentFields(queryConfig.getDocumentFields()); + } + s.setAmount(queryConfig.getAmount()); if (queryConfig.getFacets() != null) { @@ -149,7 +192,7 @@ protected Search buildSearch(String indexName, QueryConfig queryConfig) { } @NotNull - protected QueryResultObject buildQueryResultObject(SearchResult searchResult, QueryConfig queryConfig) { + protected QueryResultObject buildQueryResultObject(SearchResult searchResult, SearchConfig queryConfig) { QueryResultObject result = new QueryResultObject(); result.count = searchResult.getTotalHits(); @@ -234,7 +277,6 @@ protected QueryResultObject buildQueryResultObject(SearchResult searchResult, Qu return statFacetValueObject; } - @NotNull protected List evaluateTestsWithQueryResults(Map resultMap) { List testResults = new ArrayList<>(); diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java index 52bd45a8..179044d2 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java @@ -1,74 +1,53 @@ package io.zulia.testing.config; -import java.util.List; - -public final class QueryConfig { - private String index; - private String query; - - private int amount; - - private List facets; +import io.zulia.message.ZuliaQuery.Query.QueryType; - private List statFacets; +import java.util.List; - private List numStats; +public class QueryConfig { + private String q; + private List qf; + private int mm; + private QueryType queryType = QueryType.SCORE_MUST; public QueryConfig() { } - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public int getAmount() { - return amount; + public String getQ() { + return q; } - public void setAmount(int amount) { - this.amount = amount; + public void setQ(String q) { + this.q = q; } - public List getFacets() { - return facets; + public List getQf() { + return qf; } - public void setFacets(List facets) { - this.facets = facets; + public void setQf(List qf) { + this.qf = qf; } - public List getStatFacets() { - return statFacets; + public int getMm() { + return mm; } - public void setStatFacets(List statFacets) { - this.statFacets = statFacets; + public void setMm(int mm) { + this.mm = mm; } - public List getNumStats() { - return numStats; + public QueryType getQueryType() { + return queryType; } - public void setNumStats(List numStats) { - this.numStats = numStats; + public void setQueryType(QueryType queryType) { + this.queryType = queryType; } @Override public String toString() { - return "QueryConfig{" + "index='" + index + '\'' + ", query='" + query + '\'' + ", amount=" + amount + ", facets=" + facets + ", statFacets=" - + statFacets + ", numStats=" + numStats + '}'; + return "QueryConfig{" + "q='" + q + '\'' + ", qf=" + qf + ", mm=" + mm + ", queryType=" + queryType + '}'; } } diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/SearchConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/SearchConfig.java new file mode 100644 index 00000000..5c76caea --- /dev/null +++ b/zulia-testing/src/main/java/io/zulia/testing/config/SearchConfig.java @@ -0,0 +1,84 @@ +package io.zulia.testing.config; + +import java.util.List; + +public final class SearchConfig { + private String index; + private List queries; + + private int amount; + + private List documentFields; + + private List facets; + + private List statFacets; + + private List numStats; + + public SearchConfig() { + + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public List getQueries() { + return queries; + } + + public void setQueries(List queries) { + this.queries = queries; + } + + public int getAmount() { + return amount; + } + + public void setAmount(int amount) { + this.amount = amount; + } + + public List getDocumentFields() { + return documentFields; + } + + public void setDocumentFields(List documentFields) { + this.documentFields = documentFields; + } + + public List getFacets() { + return facets; + } + + public void setFacets(List facets) { + this.facets = facets; + } + + public List getStatFacets() { + return statFacets; + } + + public void setStatFacets(List statFacets) { + this.statFacets = statFacets; + } + + public List getNumStats() { + return numStats; + } + + public void setNumStats(List numStats) { + this.numStats = numStats; + } + + @Override + public String toString() { + return "SearchConfig{" + "index='" + index + '\'' + ", queries=" + queries + ", amount=" + amount + ", documentFields=" + documentFields + ", facets=" + + facets + ", statFacets=" + statFacets + ", numStats=" + numStats + '}'; + } +} diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java index 7d6132ef..8017f33a 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java @@ -6,7 +6,7 @@ public class ZuliaTestConfig { private Map connections; private Map indexes; - private Map searches; + private Map searches; private Map tests; private boolean logSearches; @@ -32,11 +32,11 @@ public void setIndexes(Map indexes) { this.indexes = indexes; } - public Map getSearches() { + public Map getSearches() { return searches; } - public void setSearches(Map searches) { + public void setSearches(Map searches) { this.searches = searches; } diff --git a/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java index 9302f4a9..495d9bfe 100644 --- a/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java +++ b/zulia-tools/src/main/java/io/zulia/tools/cmd/ZuliaTest.java @@ -48,12 +48,20 @@ public Integer call() throws Exception { LOG.info("Writing results to <" + testOutput + ">"); FileDataOutputStream dataOutputStream = FileDataOutputStream.from(testOutput, true); + + boolean anyFailed = false; try (CSVDataTarget csvDataTarget = CSVDataTarget.withDefaults(dataOutputStream)) { for (TestResult testResult : testResults) { csvDataTarget.writeRow(testResult.getTestId(), testResult.isPassed() ? "PASS" : "FAIL"); + if (!testResult.isPassed()) { + anyFailed = true; + } } } + if (anyFailed) { + System.exit(1); + } } } From 5ef259e5fbe35cadfc45d9a14a73a49530f847c8 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Sun, 7 Apr 2024 13:36:30 -0400 Subject: [PATCH 08/12] update sample config --- .../src/test/resources/sample_testing.yaml | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/zulia-testing/src/test/resources/sample_testing.yaml b/zulia-testing/src/test/resources/sample_testing.yaml index f522d2e6..e4d037a9 100644 --- a/zulia-testing/src/test/resources/sample_testing.yaml +++ b/zulia-testing/src/test/resources/sample_testing.yaml @@ -11,13 +11,17 @@ indexes: searches: pubsStar: index: publications - query: "*:*" + queries: + - q: "*:*" + queryType: "FILTER" pubsWithoutTitle: index: publications - query: "-title:*" + queries: + - q: "-title:*" pubsComplex: index: publications - query: "title:*" + queries: + - q: "title:*" facets: - field: pubYear topN: 10 @@ -29,14 +33,18 @@ searches: topN: 10 pubsComplex2: index: publications - query: "title:cancer" + queries: + - q: "cancer" + qf: [ "title","abstract" ] numStats: - numericField: pubYear percentiles: [ 0.1, 0.5, 0.9 ] pubsWithDocument: index: publications - query: "title:\"Understanding PCSK9 and anti-PCSK9 therapies\"" + queries: + - q: "title:\"Understanding PCSK9 and anti-PCSK9 therapies\"" amount: 1 + documentFields: [ "authors" ] tests: pubsCount: expr: pubsStar.count > 30000000 @@ -55,5 +63,5 @@ tests: numStatCheck: expr: (pubsComplex2.numStat["pubYear"].sum/pubsComplex2.numStat["pubYear"].docCount) > 2008 numStatCheck2: - expr: (pubsComplex2.numStat["pubYear"].percentiles[0].value < 1990) && (pubsComplex2.numStat["pubYear"].percentiles[2].value > 2021) + expr: (pubsComplex2.numStat["pubYear"].percentiles[0].value < 1996) && (pubsComplex2.numStat["pubYear"].percentiles[2].value > 2021) From 9f02b6f0435a012db94cdc093640b25153eba101 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Mon, 8 Apr 2024 13:18:53 -0400 Subject: [PATCH 09/12] switch to declaring project dependencies using the type-safe API --- settings.gradle | 2 +- zulia-analyzer/build.gradle.kts | 2 +- zulia-client/build.gradle.kts | 4 ++-- zulia-cmd-shared/build.gradle.kts | 2 +- zulia-common/build.gradle.kts | 2 +- zulia-data/build.gradle.kts | 2 +- zulia-query-parser/build.gradle.kts | 2 +- zulia-server/build.gradle.kts | 4 ++-- zulia-testing/build.gradle.kts | 4 ++-- zulia-tools/build.gradle.kts | 8 ++++---- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/settings.gradle b/settings.gradle index 472aac55..015f9a12 100644 --- a/settings.gradle +++ b/settings.gradle @@ -2,4 +2,4 @@ rootProject.name = "ZuliaSearch" include 'zulia-analyzer', 'zulia-common', 'zulia-cmd-shared', 'zulia-client', 'zulia-data', 'zulia-query-parser', 'zulia-server', 'zulia-testing', 'zulia-tools', 'zulia-util' - +enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") diff --git a/zulia-analyzer/build.gradle.kts b/zulia-analyzer/build.gradle.kts index 24941751..b0489aab 100644 --- a/zulia-analyzer/build.gradle.kts +++ b/zulia-analyzer/build.gradle.kts @@ -9,7 +9,7 @@ val luceneVersion: String by project defaultTasks("build", "installDist") dependencies { - api(project(":zulia-common")) + api(projects.zuliaCommon) api(libs.lucene.analysis.common) } diff --git a/zulia-client/build.gradle.kts b/zulia-client/build.gradle.kts index f88aa5c2..c8bf0d83 100644 --- a/zulia-client/build.gradle.kts +++ b/zulia-client/build.gradle.kts @@ -8,8 +8,8 @@ val okHttpVersion: String by project val gsonVersion: String by project dependencies { - api(project(":zulia-common")) - api(project(":zulia-util")) + api(projects.zuliaCommon) + api(projects.zuliaUtil) implementation(libs.unirest.java.bom) implementation(libs.unirest.core) implementation(libs.unirest.gson) diff --git a/zulia-cmd-shared/build.gradle.kts b/zulia-cmd-shared/build.gradle.kts index 734849eb..2081c75f 100644 --- a/zulia-cmd-shared/build.gradle.kts +++ b/zulia-cmd-shared/build.gradle.kts @@ -6,7 +6,7 @@ description = "Zulia CMD Shared" dependencies { annotationProcessor(libs.picocli.codegen) - api(project(":zulia-common")) + api(projects.zuliaCommon) implementation(libs.picocli.base) } diff --git a/zulia-common/build.gradle.kts b/zulia-common/build.gradle.kts index 20720358..6b2942c5 100644 --- a/zulia-common/build.gradle.kts +++ b/zulia-common/build.gradle.kts @@ -23,7 +23,7 @@ dependencies { protobuf(libs.sketches.java) - api(project(":zulia-util")) + api(projects.zuliaUtil) } diff --git a/zulia-data/build.gradle.kts b/zulia-data/build.gradle.kts index 86311e6d..c0ed3b9c 100644 --- a/zulia-data/build.gradle.kts +++ b/zulia-data/build.gradle.kts @@ -9,7 +9,7 @@ dependencies { api(libs.bundles.poi) api(libs.univocity.parsers) api(libs.jackson.databind) - api(project(":zulia-util")) + api(projects.zuliaUtil) } diff --git a/zulia-query-parser/build.gradle.kts b/zulia-query-parser/build.gradle.kts index 8078a656..0c531cf6 100644 --- a/zulia-query-parser/build.gradle.kts +++ b/zulia-query-parser/build.gradle.kts @@ -9,7 +9,7 @@ val luceneVersion: String by project defaultTasks("build", "installDist") dependencies { - api(project(":zulia-analyzer")) + api(projects.zuliaAnalyzer) api(libs.lucene.queryparser) } diff --git a/zulia-server/build.gradle.kts b/zulia-server/build.gradle.kts index 7ab6e471..60fcff86 100644 --- a/zulia-server/build.gradle.kts +++ b/zulia-server/build.gradle.kts @@ -37,8 +37,8 @@ tasks.withType { } dependencies { - implementation(project(":zulia-query-parser")) - implementation(project(":zulia-cmd-shared")) + implementation(projects.zuliaQueryParser) + implementation(projects.zuliaCmdShared) annotationProcessor(libs.micronaut.http.validation) annotationProcessor(libs.micronaut.openapi) annotationProcessor(libs.micronaut.serde.processor) diff --git a/zulia-testing/build.gradle.kts b/zulia-testing/build.gradle.kts index ff1372d8..48571f0a 100644 --- a/zulia-testing/build.gradle.kts +++ b/zulia-testing/build.gradle.kts @@ -6,8 +6,8 @@ description = "Zulia Testing" dependencies { - api(project(":zulia-util")) - api(project(":zulia-client")) + api(projects.zuliaUtil) + api(projects.zuliaClient) implementation(libs.graalvm.js) implementation(libs.logback.classic) implementation(libs.jansi) diff --git a/zulia-tools/build.gradle.kts b/zulia-tools/build.gradle.kts index bb404e0e..9c9e076d 100644 --- a/zulia-tools/build.gradle.kts +++ b/zulia-tools/build.gradle.kts @@ -7,10 +7,10 @@ description = "Zulia Tools" dependencies { annotationProcessor(libs.picocli.codegen) - implementation(project(":zulia-client")) - implementation(project(":zulia-cmd-shared")) - implementation(project(":zulia-testing")) - implementation(project(":zulia-data")) + implementation(projects.zuliaClient) + implementation(projects.zuliaCmdShared) + implementation(projects.zuliaTesting) + implementation(projects.zuliaData) implementation(libs.snake.yaml) implementation(libs.picocli.base) implementation(libs.commons.compress) From f81222247feabbafdf352f57fa293c383bc9dec0 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Sun, 14 Apr 2024 23:25:22 -0400 Subject: [PATCH 10/12] refactor to allow yaml schema --- .../io/zulia/testing/ZuliaTestRunner.java | 66 +++++++++---------- .../testing/config/ConnectionConfig.java | 12 +++- .../io/zulia/testing/config/IndexConfig.java | 12 +++- .../io/zulia/testing/config/QueryConfig.java | 1 + .../io/zulia/testing/config/SearchConfig.java | 14 +++- .../io/zulia/testing/config/TestConfig.java | 13 +++- .../zulia/testing/config/ZuliaTestConfig.java | 26 ++++---- .../src/test/resources/sample_testing.yaml | 40 ++++++----- 8 files changed, 116 insertions(+), 68 deletions(-) diff --git a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java index 4962ae08..3a6fe89e 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java +++ b/zulia-testing/src/main/java/io/zulia/testing/ZuliaTestRunner.java @@ -44,6 +44,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; import static io.zulia.message.ZuliaQuery.Query.QueryType.FILTER; import static io.zulia.message.ZuliaQuery.Query.QueryType.FILTER_NOT; @@ -68,26 +70,23 @@ public List runTests() throws Exception { @NotNull protected Map> buildConnectionSupplier() { - Map connectionIdToConnectionConfig = zuliaTestConfig.getConnections(); - Map indexNameToIndexConfig = zuliaTestConfig.getIndexes(); + List connections = zuliaTestConfig.getConnections(); + List indexConfigs = zuliaTestConfig.getIndexes(); Map> connectionToConnectionConfig = new HashMap<>(); - for (Map.Entry indexConfigEntry : connectionIdToConnectionConfig.entrySet()) { - ConnectionConfig connectionConfig = indexConfigEntry.getValue(); - + for (ConnectionConfig connectionConfig : connections) { ThrowingSupplier throwingSupplier = () -> { ZuliaPoolConfig zuliaPoolConfig = new ZuliaPoolConfig().addNode(connectionConfig.getServerAddress(), connectionConfig.getPort()); return new ZuliaWorkPool(zuliaPoolConfig); }; - connectionToConnectionConfig.put(indexConfigEntry.getKey(), Suppliers.memoize(throwingSupplier)); + connectionToConnectionConfig.put(connectionConfig.getName(), Suppliers.memoize(throwingSupplier)); } - for (Map.Entry indexConfigEntry : indexNameToIndexConfig.entrySet()) { - IndexConfig indexConfig = indexConfigEntry.getValue(); + for (IndexConfig indexConfig : indexConfigs) { Supplier zuliaWorkPoolSupplier = connectionToConnectionConfig.get(indexConfig.getConnection()); if (zuliaWorkPoolSupplier == null) { throw new IllegalArgumentException( - "Failed to find connection config <" + indexConfig.getConnection() + "> for query <" + indexConfigEntry.getKey() + ">"); + "Failed to find connection config <" + indexConfig.getConnection() + "> for index config <" + indexConfig.getName() + ">"); } } return connectionToConnectionConfig; @@ -95,29 +94,29 @@ protected Map> buildConnectionSupplier() { @NotNull protected Map buildAndRunQueries() throws Exception { - Map searches = zuliaTestConfig.getSearches(); - Map indexNameToIndexConfig = zuliaTestConfig.getIndexes(); + List searches = zuliaTestConfig.getSearches(); + Map indexNameToIndexConfig = zuliaTestConfig.getIndexes().stream() + .collect(Collectors.toMap(IndexConfig::getName, Function.identity())); Map resultMap = new HashMap<>(); - for (Map.Entry queryConfigEntry : searches.entrySet()) { - SearchConfig queryConfig = queryConfigEntry.getValue(); - IndexConfig indexConfig = indexNameToIndexConfig.get(queryConfig.getIndex()); + for (SearchConfig searchConfig : searches) { + IndexConfig indexConfig = indexNameToIndexConfig.get(searchConfig.getIndex()); ZuliaWorkPool zuliaWorkPool = connectionToConnectionConfig.get(indexConfig.getConnection()).get(); - Search s = buildSearch(indexConfig.getIndexName(), queryConfig); + Search s = buildSearch(indexConfig.getIndexName(), searchConfig); if (zuliaTestConfig.isLogSearches()) { - LOG.info("Running search <" + queryConfigEntry.getKey() + ">:\n" + s); + LOG.info("Running search <" + searchConfig.getName() + ">:\n" + s); } SearchResult searchResult = zuliaWorkPool.search(s); - QueryResultObject result = buildQueryResultObject(searchResult, queryConfig); - resultMap.put(queryConfigEntry.getKey(), result); + QueryResultObject result = buildQueryResultObject(searchResult, searchConfig); + resultMap.put(searchConfig.getName(), result); } return resultMap; } @NotNull - protected Search buildSearch(String indexName, SearchConfig queryConfig) { + protected Search buildSearch(String indexName, SearchConfig searchConfig) { Search s = new Search(indexName); - List queries = queryConfig.getQueries(); + List queries = searchConfig.getQueries(); if (queries != null) { for (QueryConfig query : queries) { if (FILTER == query.getQueryType() || FILTER_NOT == query.getQueryType()) { @@ -150,14 +149,14 @@ else if (SCORE_MUST == query.getQueryType() || SCORE_SHOULD == query.getQueryTyp } } - if (queryConfig.getDocumentFields() != null) { - s.addDocumentFields(queryConfig.getDocumentFields()); + if (searchConfig.getDocumentFields() != null) { + s.addDocumentFields(searchConfig.getDocumentFields()); } - s.setAmount(queryConfig.getAmount()); + s.setAmount(searchConfig.getAmount()); - if (queryConfig.getFacets() != null) { - for (FacetConfig facet : queryConfig.getFacets()) { + if (searchConfig.getFacets() != null) { + for (FacetConfig facet : searchConfig.getFacets()) { CountFacet countFacet = new CountFacet(facet.getField()); if (facet.getTopN() != 0) { countFacet.setTopN(facet.getTopN()); @@ -166,8 +165,8 @@ else if (SCORE_MUST == query.getQueryType() || SCORE_SHOULD == query.getQueryTyp } } - if (queryConfig.getStatFacets() != null) { - for (StatFacetConfig statFacetConfig : queryConfig.getStatFacets()) { + if (searchConfig.getStatFacets() != null) { + for (StatFacetConfig statFacetConfig : searchConfig.getStatFacets()) { StatFacet statFacet = new StatFacet(statFacetConfig.getNumericField(), statFacetConfig.getFacetField()); if (statFacetConfig.getTopN() != 0) { statFacet.setTopN(statFacetConfig.getTopN()); @@ -175,8 +174,8 @@ else if (SCORE_MUST == query.getQueryType() || SCORE_SHOULD == query.getQueryTyp s.addStat(statFacet); } } - if (queryConfig.getNumStats() != null) { - for (NumStatConfig numStatConfig : queryConfig.getNumStats()) { + if (searchConfig.getNumStats() != null) { + for (NumStatConfig numStatConfig : searchConfig.getNumStats()) { NumericStat numericStat = new NumericStat(numStatConfig.getNumericField()); if (numStatConfig.getPercentilePrecision() > 0) { numericStat.setPercentilePrecision(numStatConfig.getPercentilePrecision()); @@ -290,16 +289,15 @@ protected List evaluateTestsWithQueryResults(Map testConfigEntry : zuliaTestConfig.getTests().entrySet()) { - LOG.info("Running Test <" + testConfigEntry.getKey() + ">"); - TestConfig testConfig = testConfigEntry.getValue(); + for (TestConfig testConfig : zuliaTestConfig.getTests()) { + LOG.info("Running Test <" + testConfig.getName() + ">"); TestResult testResult = new TestResult(); - testResult.setTestId(testConfigEntry.getKey()); + testResult.setTestId(testConfig.getName()); testResult.setTestConfig(testConfig); Value js = context.eval("js", testConfig.getExpr()); testResult.setPassed(js.asBoolean()); - LOG.info("Test <" + testConfigEntry.getKey() + "> " + (js.asBoolean() ? "Passed" : "Failed")); + LOG.info("Test <" + testConfig.getName() + "> " + (js.asBoolean() ? "Passed" : "Failed")); testResults.add(testResult); } diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java index 5285849d..c616d73f 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java @@ -2,6 +2,8 @@ public class ConnectionConfig { + private String name; + private String serverAddress; private int port; @@ -10,6 +12,14 @@ public ConnectionConfig() { } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + public String getServerAddress() { return serverAddress; } @@ -28,7 +38,7 @@ public void setPort(int port) { @Override public String toString() { - return "ConnectionConfig{" + "serverAddress='" + serverAddress + '\'' + ", port=" + port + '}'; + return "ConnectionConfig{" + "name='" + name + '\'' + ", serverAddress='" + serverAddress + '\'' + ", port=" + port + '}'; } } diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/IndexConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/IndexConfig.java index 1b84fd60..db470d12 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/IndexConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/IndexConfig.java @@ -2,6 +2,8 @@ public final class IndexConfig { + private String name; + private String indexName; private String connection; @@ -9,6 +11,14 @@ public final class IndexConfig { public IndexConfig() { } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + public String getIndexName() { return indexName; } @@ -27,6 +37,6 @@ public void setConnection(String connection) { @Override public String toString() { - return "IndexConfig{" + "indexName='" + indexName + '\'' + ", connection='" + connection + '\'' + '}'; + return "IndexConfig{" + "name='" + name + '\'' + ", indexName='" + indexName + '\'' + ", connection='" + connection + '\'' + '}'; } } diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java index 179044d2..241fc2a6 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/QueryConfig.java @@ -5,6 +5,7 @@ import java.util.List; public class QueryConfig { + private String q; private List qf; private int mm; diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/SearchConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/SearchConfig.java index 5c76caea..17416b6e 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/SearchConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/SearchConfig.java @@ -3,6 +3,8 @@ import java.util.List; public final class SearchConfig { + private String name; + private String index; private List queries; @@ -20,6 +22,14 @@ public SearchConfig() { } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + public String getIndex() { return index; } @@ -78,7 +88,7 @@ public void setNumStats(List numStats) { @Override public String toString() { - return "SearchConfig{" + "index='" + index + '\'' + ", queries=" + queries + ", amount=" + amount + ", documentFields=" + documentFields + ", facets=" - + facets + ", statFacets=" + statFacets + ", numStats=" + numStats + '}'; + return "SearchConfig{" + "name='" + name + '\'' + ", index='" + index + '\'' + ", queries=" + queries + ", amount=" + amount + ", documentFields=" + + documentFields + ", facets=" + facets + ", statFacets=" + statFacets + ", numStats=" + numStats + '}'; } } diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/TestConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/TestConfig.java index 455ea466..78063f22 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/TestConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/TestConfig.java @@ -1,12 +1,23 @@ package io.zulia.testing.config; public final class TestConfig { + + private String name; + private String expr; public TestConfig() { } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + public String getExpr() { return expr; } @@ -17,6 +28,6 @@ public void setExpr(String expr) { @Override public String toString() { - return "TestConfig{" + "expr='" + expr + '\'' + '}'; + return "TestConfig{" + "name='" + name + '\'' + ", expr='" + expr + '\'' + '}'; } } diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java index 8017f33a..4411b4e8 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/ZuliaTestConfig.java @@ -1,13 +1,13 @@ package io.zulia.testing.config; -import java.util.Map; +import java.util.List; public class ZuliaTestConfig { - private Map connections; - private Map indexes; - private Map searches; - private Map tests; + private List connections; + private List indexes; + private List searches; + private List tests; private boolean logSearches; private boolean logSearchResults; @@ -16,35 +16,35 @@ public ZuliaTestConfig() { } - public Map getConnections() { + public List getConnections() { return connections; } - public void setConnections(Map connections) { + public void setConnections(List connections) { this.connections = connections; } - public Map getIndexes() { + public List getIndexes() { return indexes; } - public void setIndexes(Map indexes) { + public void setIndexes(List indexes) { this.indexes = indexes; } - public Map getSearches() { + public List getSearches() { return searches; } - public void setSearches(Map searches) { + public void setSearches(List searches) { this.searches = searches; } - public Map getTests() { + public List getTests() { return tests; } - public void setTests(Map tests) { + public void setTests(List tests) { this.tests = tests; } diff --git a/zulia-testing/src/test/resources/sample_testing.yaml b/zulia-testing/src/test/resources/sample_testing.yaml index e4d037a9..41ef4105 100644 --- a/zulia-testing/src/test/resources/sample_testing.yaml +++ b/zulia-testing/src/test/resources/sample_testing.yaml @@ -1,24 +1,31 @@ +# yaml-language-server: $schema=https://maven.zulia.io/schema/zulia-test.json logSearches: true logSearchResults: true connections: - testConnection: + - name: testConnection serverAddress: localhost port: 32191 + - name: connection2 + port: 32191 + serverAddress: somewhere + - name: blah + serverAddress: blahblah + port: 32191 indexes: - publications: + - name: publications indexName: publications connection: testConnection searches: - pubsStar: + - name: pubsStar index: publications queries: - q: "*:*" queryType: "FILTER" - pubsWithoutTitle: + - name: pubsWithoutTitle index: publications queries: - q: "-title:*" - pubsComplex: + - name: pubsComplex index: publications queries: - q: "title:*" @@ -27,11 +34,12 @@ searches: topN: 10 - field: journalNameIso topN: 10 + statFacets: - facetField: pubYear numericField: authorCount topN: 10 - pubsComplex2: + - name: pubsComplex2 index: publications queries: - q: "cancer" @@ -39,29 +47,29 @@ searches: numStats: - numericField: pubYear percentiles: [ 0.1, 0.5, 0.9 ] - pubsWithDocument: + - name: pubsWithDocument index: publications queries: - q: "title:\"Understanding PCSK9 and anti-PCSK9 therapies\"" amount: 1 documentFields: [ "authors" ] tests: - pubsCount: + - name: pubsCount expr: pubsStar.count > 30000000 - mostPubsHaveTitle: + - name: mostPubsHaveTitle expr: pubsWithoutTitle.count < pubsStar.count * 0.01 - oneResult: + - name: oneResult expr: pubsWithDocument.count == 1 - resultHasCorrectAuthor: + - name: resultHasCorrectAuthor expr: pubsWithDocument.doc[0]["authors"][0]["lastName"] == "McKenney" - topYearFacet: + - name: topYearFacet expr: pubsComplex.facet["pubYear"][0].label == "2022" && pubsComplex.facet["pubYear"][0].count > 1000000 - topJournalFacet: + - name: topJournalFacet expr: pubsComplex.facet["journalNameIso"][0].label = "PLoS One" && pubsComplex.facet["journalNameIso"][0].count > 1000 - statFacetCheck: + - name: statFacetCheck expr: pubsComplex.statFacet["pubYear-authorCount"][0].label = "2022" && pubsComplex.statFacet["pubYear-authorCount"][0].docCount > 1000000 && pubsComplex.statFacet["pubYear-authorCount"][0].sum > 5000000 - numStatCheck: + - name: numStatCheck expr: (pubsComplex2.numStat["pubYear"].sum/pubsComplex2.numStat["pubYear"].docCount) > 2008 - numStatCheck2: + - name: numStatCheck2 expr: (pubsComplex2.numStat["pubYear"].percentiles[0].value < 1996) && (pubsComplex2.numStat["pubYear"].percentiles[2].value > 2021) From e89bf0df3c72daf1b943994db534856385e61b4a Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Thu, 18 Apr 2024 11:56:53 -0400 Subject: [PATCH 11/12] update schema to final 1.0 version --- .../testing/config/ConnectionConfig.java | 2 +- .../resources/zulia_testing_schema-1.0.json | 242 ++++++++++++++++++ .../src/test/resources/sample_testing.yaml | 3 +- 3 files changed, 244 insertions(+), 3 deletions(-) create mode 100644 zulia-testing/src/main/resources/zulia_testing_schema-1.0.json diff --git a/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java b/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java index c616d73f..92bb2727 100644 --- a/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java +++ b/zulia-testing/src/main/java/io/zulia/testing/config/ConnectionConfig.java @@ -6,7 +6,7 @@ public class ConnectionConfig { private String serverAddress; - private int port; + private int port = 32191; public ConnectionConfig() { diff --git a/zulia-testing/src/main/resources/zulia_testing_schema-1.0.json b/zulia-testing/src/main/resources/zulia_testing_schema-1.0.json new file mode 100644 index 00000000..6a5428b4 --- /dev/null +++ b/zulia-testing/src/main/resources/zulia_testing_schema-1.0.json @@ -0,0 +1,242 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "#/definitions/ZuliaTesting", + "definitions": { + "ZuliaTesting": { + "type": "object", + "additionalProperties": false, + "properties": { + "logSearches": { + "type": "boolean" + }, + "logSearchResults": { + "type": "boolean" + }, + "connections": { + "type": "array", + "items": { + "$ref": "#/definitions/Connection" + } + }, + "indexes": { + "type": "array", + "items": { + "$ref": "#/definitions/Index" + } + }, + "searches": { + "type": "array", + "items": { + "$ref": "#/definitions/Search" + } + }, + "tests": { + "type": "array", + "items": { + "$ref": "#/definitions/Test" + } + } + }, + "required": [ + "connections", + "indexes", + "logSearchResults", + "logSearches", + "searches", + "tests" + ], + "title": "ZuliaTesting" + }, + "Connection": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "serverAddress": { + "type": "string" + }, + "port": { + "type": "integer" + } + }, + "required": [ + "name", + "serverAddress" + ], + "title": "Connection" + }, + "Index": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "indexName": { + "type": "string" + }, + "connection": { + "type": "string" + } + }, + "required": [ + "connection", + "indexName", + "name" + ], + "title": "Index" + }, + "Search": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "index": { + "type": "string" + }, + "queries": { + "type": "array", + "items": { + "$ref": "#/definitions/Query" + } + }, + "facets": { + "type": "array", + "items": { + "$ref": "#/definitions/Facet" + } + }, + "statFacets": { + "type": "array", + "items": { + "$ref": "#/definitions/StatFacet" + } + }, + "numStats": { + "type": "array", + "items": { + "$ref": "#/definitions/NumStat" + } + }, + "amount": { + "type": "integer" + }, + "documentFields": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "index", + "name", + "queries" + ], + "title": "Search" + }, + "Facet": { + "type": "object", + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "topN": { + "type": "integer" + } + }, + "required": [ + "field", + "topN" + ], + "title": "Facet" + }, + "NumStat": { + "type": "object", + "additionalProperties": false, + "properties": { + "numericField": { + "type": "string" + }, + "percentilePrecision": { + "type": "number" + }, + "percentiles": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "required": [ + "numericField" + ], + "title": "NumStat" + }, + "Query": { + "type": "object", + "additionalProperties": false, + "properties": { + "q": { + "type": "string" + }, + "queryType": { + "type": "string" + }, + "qf": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "q" + ], + "title": "Query" + }, + "StatFacet": { + "type": "object", + "additionalProperties": false, + "properties": { + "facetField": { + "type": "string" + }, + "numericField": { + "type": "string" + }, + "topN": { + "type": "integer" + } + }, + "required": [ + "facetField", + "numericField", + "topN" + ], + "title": "StatFacet" + }, + "Test": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "expr": { + "type": "string" + } + }, + "required": [ + "expr", + "name" + ], + "title": "Test" + } + } +} \ No newline at end of file diff --git a/zulia-testing/src/test/resources/sample_testing.yaml b/zulia-testing/src/test/resources/sample_testing.yaml index 41ef4105..fab7f151 100644 --- a/zulia-testing/src/test/resources/sample_testing.yaml +++ b/zulia-testing/src/test/resources/sample_testing.yaml @@ -1,10 +1,9 @@ -# yaml-language-server: $schema=https://maven.zulia.io/schema/zulia-test.json +# yaml-language-server: $schema=https://maven.zulia.io/schema/zulia_testing_schema-1.0.json logSearches: true logSearchResults: true connections: - name: testConnection serverAddress: localhost - port: 32191 - name: connection2 port: 32191 serverAddress: somewhere From cfc5b5004cf746b9cbacdd246f1962f735b97697 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Thu, 18 Apr 2024 12:27:37 -0400 Subject: [PATCH 12/12] update schema to include enum for query type and min match (mm) --- ...g_schema-1.0.json => zulia_testing_schema-1.1.json} | 10 +++++++++- zulia-testing/src/test/resources/sample_testing.yaml | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) rename zulia-testing/src/main/resources/{zulia_testing_schema-1.0.json => zulia_testing_schema-1.1.json} (96%) diff --git a/zulia-testing/src/main/resources/zulia_testing_schema-1.0.json b/zulia-testing/src/main/resources/zulia_testing_schema-1.1.json similarity index 96% rename from zulia-testing/src/main/resources/zulia_testing_schema-1.0.json rename to zulia-testing/src/main/resources/zulia_testing_schema-1.1.json index 6a5428b4..65db487c 100644 --- a/zulia-testing/src/main/resources/zulia_testing_schema-1.0.json +++ b/zulia-testing/src/main/resources/zulia_testing_schema-1.1.json @@ -186,13 +186,21 @@ "type": "string" }, "queryType": { - "type": "string" + "enum": [ + "FILTER", + "FILTER_NOT", + "SCORE_MUST", + "SCORE_SHOULD" + ] }, "qf": { "type": "array", "items": { "type": "string" } + }, + "mm": { + "type": "number" } }, "required": [ diff --git a/zulia-testing/src/test/resources/sample_testing.yaml b/zulia-testing/src/test/resources/sample_testing.yaml index fab7f151..224f9b2f 100644 --- a/zulia-testing/src/test/resources/sample_testing.yaml +++ b/zulia-testing/src/test/resources/sample_testing.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://maven.zulia.io/schema/zulia_testing_schema-1.0.json +# yaml-language-server: $schema=https://maven.zulia.io/schema/zulia_testing_schema-1.1.json logSearches: true logSearchResults: true connections: