Skip to content

Commit

Permalink
Rework tests and test suite. Add new test suite for testing in-develo…
Browse files Browse the repository at this point in the history
…pment JDBC driver.

Signed-off-by: Yury-Fridlyand <[email protected]>
  • Loading branch information
Yury-Fridlyand committed Jan 24, 2023
1 parent 24d3767 commit 24de6ff
Show file tree
Hide file tree
Showing 3 changed files with 288 additions and 32 deletions.
126 changes: 125 additions & 1 deletion integ-test/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,10 @@ compileTestJava {

testClusters.all {
testDistribution = 'archive'
plugin ":opensearch-sql-plugin"
}

testClusters.integTest {
plugin ":opensearch-sql-plugin"
keystore 'plugins.query.federation.datasources.config', new File("$projectDir/src/test/resources/datasource/", 'datasources.json')
}

Expand Down Expand Up @@ -148,8 +148,113 @@ task stopPrometheus(type: KillProcessTask) {

stopPrometheus.mustRunAfter startPrometheus

task deleteJdbcDriverRepo {
// Delete local clone of the sql-jdbc repo
file("sql-jdbc").deleteDir()
}

task cloneJdbcDriverRepo {
doFirst {
file("sql-jdbc").deleteDir()

exec {
// clone the sql-jdbc repo locally
commandLine 'git', 'clone', System.getProperty('jdbcRepo', 'https://github.com/opensearch-project/sql-jdbc.git')
}
exec {
workingDir 'sql-jdbc'
commandLine 'git', 'switch', System.getProperty("jdbcBranch", 'main')
}
// TODO would fail on windows
exec {
workingDir 'sql-jdbc'
commandLine 'sh', 'gradlew', 'shadowJar'
}
}
}

task integJdbcTest(type: RestIntegTestTask) {
useJUnitPlatform()
dependsOn ':opensearch-sql-plugin:bundlePlugin'
testLogging {
events "passed", "skipped", "failed"
}
afterTest { desc, result ->
logger.quiet "${desc.className}.${desc.name}: ${result.resultType} ${(result.getEndTime() - result.getStartTime())/1000}s"
}

systemProperty 'tests.security.manager', 'false'
systemProperty('project.root', project.projectDir.absolutePath)

systemProperty "https", System.getProperty("https")
systemProperty "user", System.getProperty("user")
systemProperty "password", System.getProperty("password")

// Set default query size limit
systemProperty 'defaultQuerySizeLimit', '10000'

// Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
// requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
doFirst { systemProperty 'cluster.debug', getDebug() }

if (System.getProperty("test.debug") != null) {
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
}

filter {
includeTestsMatching '*.jdbc.*'
}
}

task integDevJdbcTest(type: RestIntegTestTask) {
useJUnitPlatform()
dependsOn ':opensearch-sql-plugin:bundlePlugin'
testLogging {
events "passed", "skipped", "failed"
}
afterTest { desc, result ->
logger.quiet "${desc.className}.${desc.name}: ${result.resultType} ${(result.getEndTime() - result.getStartTime())/1000}s"
}

if (System.getProperty("jdbcFile") != null) {
file("sql-jdbc/build/libs").mkdirs()
copy {
from System.getProperty("jdbcFile")
into "sql-jdbc/build/libs"
}
} else {
dependsOn cloneJdbcDriverRepo
}
finalizedBy deleteJdbcDriverRepo

systemProperty 'tests.security.manager', 'false'
systemProperty('project.root', project.projectDir.absolutePath)

systemProperty "https", System.getProperty("https")
systemProperty "user", System.getProperty("user")
systemProperty "password", System.getProperty("password")

// Set default query size limit
systemProperty 'defaultQuerySizeLimit', '10000'

// Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
// requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
doFirst { systemProperty 'cluster.debug', getDebug() }

if (System.getProperty("test.debug") != null) {
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
}

filter {
includeTestsMatching '*.devJdbc.*'
}
}

// Run PPL ITs and new, legacy and comparison SQL ITs with new SQL engine enabled
integTest {
testLogging {
events "passed", "skipped", "failed"
}
dependsOn ':opensearch-sql-plugin:bundlePlugin'
dependsOn startPrometheus
finalizedBy stopPrometheus
Expand Down Expand Up @@ -192,10 +297,17 @@ integTest {

// Skip this IT because all assertions are against explain output
exclude 'org/opensearch/sql/legacy/OrderIT.class'

// Exclude JDBC related tests
exclude '**/jdbc/**'
exclude '**/devJdbc/**'
}


task comparisonTest(type: RestIntegTestTask) {
testLogging {
events "passed", "skipped", "failed"
}
dependsOn ':opensearch-sql-plugin:bundlePlugin'

systemProperty 'tests.security.manager', 'false'
Expand All @@ -214,6 +326,10 @@ task comparisonTest(type: RestIntegTestTask) {
exclude 'org/opensearch/sql/ppl/**/*IT.class'
exclude 'org/opensearch/sql/legacy/**/*IT.class'

// Exclude JDBC related tests
exclude '**/jdbc/**'
exclude '**/devJdbc/**'

// Enable logging output to console
testLogging.showStandardStreams true

Expand Down Expand Up @@ -368,6 +484,9 @@ task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {

// A bwc test suite which runs all the bwc tasks combined
task bwcTestSuite(type: StandaloneRestIntegTestTask) {
testLogging {
events "passed", "skipped", "failed"
}
exclude '**/*Test*'
exclude '**/*IT*'
dependsOn tasks.named("${baseName}#mixedClusterTask")
Expand All @@ -379,6 +498,9 @@ def opensearch_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()

task integTestRemote(type: RestIntegTestTask) {
testLogging {
events "passed", "skipped", "failed"
}
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
systemProperty 'tests.security.manager', 'false'
Expand Down Expand Up @@ -406,4 +528,6 @@ task integTestRemote(type: RestIntegTestTask) {
exclude 'org/opensearch/sql/legacy/TermQueryExplainIT.class'
exclude 'org/opensearch/sql/legacy/QueryAnalysisIT.class'
exclude 'org/opensearch/sql/legacy/OrderIT.class'
exclude '**/jdbc/**'
exclude '**/devJdbc/**'
}
129 changes: 129 additions & 0 deletions integ-test/src/test/java/org/opensearch/sql/devJdbc/CursorIT.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.devJdbc;

import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS;
import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT;
import static org.opensearch.sql.util.TestUtils.getResponseBody;

import java.io.File;
import java.net.URL;
import java.net.URLClassLoader;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.List;
import javax.annotation.Nullable;
import lombok.SneakyThrows;
import org.json.JSONObject;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.opensearch.client.Request;
import org.opensearch.client.RequestOptions;
import org.opensearch.client.Response;
import org.opensearch.sql.legacy.SQLIntegTestCase;

public class CursorIT extends SQLIntegTestCase {

private static Connection connection;
private static Driver driver;
private boolean initialized = false;

@BeforeEach
@SneakyThrows
public void init() {
if (!initialized) {
initClient();
resetQuerySizeLimit();
loadIndex(Index.BANK);
loadIndex(Index.CALCS);
loadIndex(Index.ONLINE);
loadIndex(Index.ACCOUNT);
initialized = true;
}
}

@BeforeAll
@BeforeClass
@SneakyThrows
public static void loadDriver() {
var buildDir = String.format("%s/sql-jdbc/build/libs", System.getProperty("project.root"));
var driverFiles = new File(buildDir).
listFiles(pathname -> pathname.getAbsolutePath().endsWith(".jar"));

Assume.assumeTrue("Driver load failed", driverFiles != null && 1 == driverFiles.length);

URLClassLoader loader = new URLClassLoader(
new URL[] { driverFiles[0].toURI().toURL() },
ClassLoader.getSystemClassLoader()
);
driver = (Driver)Class.forName("org.opensearch.jdbc.Driver", true, loader)
.getDeclaredConstructor().newInstance();
connection = driver.connect(getConnectionString(), null);
}

@AfterAll
@AfterClass
@SneakyThrows
public static void closeConnection() {
// TODO should we close Statement and ResultSet?
if (connection != null) {
connection.close();
connection = null;
}
}

@Test
@SneakyThrows
public void dev_select_all_small_table_small_cursor() {
Statement stmt = connection.createStatement();

for (var table : List.of(TEST_INDEX_CALCS, TEST_INDEX_BANK)) {
var query = String.format("SELECT * FROM %s", table);
stmt.setFetchSize(3);
ResultSet rs = stmt.executeQuery(query);
int rows = 0;
for (; rs.next(); rows++) ;

var restResponse = executeRestQuery(query, null);
assertEquals(rows, restResponse.getInt("total"));
}
}

/**
* Use OpenSearch cluster initialized by OpenSearch Gradle task.
*/
private static String getConnectionString() {
// string like "[::1]:46751,127.0.0.1:34403"
var clusterUrls = System.getProperty("tests.rest.cluster").split(",");
return String.format("jdbc:opensearch://%s", clusterUrls[clusterUrls.length - 1]);
}

@SneakyThrows
protected JSONObject executeRestQuery(String query, @Nullable Integer fetch_size) {
Request request = new Request("POST", QUERY_API_ENDPOINT);
if (fetch_size != null) {
request.setJsonEntity(String.format("{ \"query\": \"%s\", \"fetch_size\": %d }", query, fetch_size));
} else {
request.setJsonEntity(String.format("{ \"query\": \"%s\" }", query));
}

RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
restOptionsBuilder.addHeader("Content-Type", "application/json");
request.setOptions(restOptionsBuilder);

Response response = client().performRequest(request);
return new JSONObject(getResponseBody(response));
}

}
Loading

0 comments on commit 24de6ff

Please sign in to comment.