From 9252b270344767f5308ac8b5fe9af81c49663706 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Mon, 13 Nov 2023 21:38:43 -0800 Subject: [PATCH 1/8] Install security plugin from individual artifacts WIP commit that tries to create a local cluster using the demo certificates that the security plugin provides. The goal of this change is to run security tests directly instead of through using the docker image. It pulls the individual artifacts from the snapshot repo. Right now, it is able to bring up a cluster. However, the wait for cluster health yellow checks are failing, so the tests are not able to run. Signed-off-by: John Mazanec --- build.gradle | 87 +++++++++++++++++++++- src/test/resources/security/admin-cert.pem | 27 +++++++ src/test/resources/security/admin-key.pem | 28 +++++++ src/test/resources/security/node-cert.pem | 25 +++++++ src/test/resources/security/node-key.pem | 28 +++++++ src/test/resources/security/root-ca.pem | 28 +++++++ 6 files changed, 219 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/security/admin-cert.pem create mode 100644 src/test/resources/security/admin-key.pem create mode 100644 src/test/resources/security/node-cert.pem create mode 100644 src/test/resources/security/node-key.pem create mode 100644 src/test/resources/security/root-ca.pem diff --git a/build.gradle b/build.gradle index 6a47fa92ca..1ac273ee6f 100644 --- a/build.gradle +++ b/build.gradle @@ -6,6 +6,8 @@ import org.opensearch.gradle.test.RestIntegTestTask import org.apache.tools.ant.taskdefs.condition.Os +import java.util.concurrent.Callable + buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set @@ -13,6 +15,19 @@ buildscript { opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + + version_tokens = opensearch_version.tokenize('-') + opensearch_build = version_tokens[0] + '.0' + plugin_no_snapshot = opensearch_build + if (version_qualifier) { + opensearch_build += "-${version_qualifier}" + plugin_no_snapshot += "-${version_qualifier}" + } + if (isSnapshot) { + opensearch_build += "-SNAPSHOT" + } + opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","") } // This isn't applying from repositories.gradle so repeating git diff it here @@ -52,7 +67,6 @@ apply plugin: 'opensearch.pluginzip' apply plugin: 'opensearch.repositories' ext { - isSnapshot = "true" == System.getProperty("build.snapshot", "true") projectSubstitutions = [:] } @@ -87,6 +101,10 @@ allprojects { } } +configurations { + zipArchive +} + publishing { repositories { maven { @@ -182,6 +200,8 @@ dependencies { testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.7' testFixturesImplementation "org.opensearch:common-utils:${version}" + + zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}" } @@ -232,9 +252,13 @@ integTest { // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) - systemProperty "https", System.getProperty("https") - systemProperty "user", System.getProperty("user") - systemProperty "password", System.getProperty("password") + systemProperty("tests.opensearch.https", "true") + systemProperty("tests.opensearch.username", "admin") + systemProperty("tests.opensearch.password", "admin") + +// systemProperty "https", System.getProperty("https") +// systemProperty "user", System.getProperty("user") +// systemProperty "password", System.getProperty("password") doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can @@ -245,6 +269,10 @@ integTest { systemProperty 'cluster.number_of_nodes', "${_numNodes}" // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written + + + + getClusters().forEach { cluster -> cluster.waitForAllConditions() } @@ -258,6 +286,57 @@ integTest { testClusters.integTest { testDistribution = "ARCHIVE" + + // Optionally install security + if (System.getProperty("security.enabled") != null) { + configurations.zipArchive.asFileTree.each { + plugin(provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it + } + } + } + })) + } + +// systemProperty("tests.opensearch.https", "true") +// systemProperty("tests.opensearch.username", "admin") +// systemProperty("tests.opensearch.password", "admin") + + systemProperty("secure", "true") + systemProperty("username", "admin") + systemProperty("password", "admin") + + extraConfigFile("admin-cert.pem", new File("$rootDir/src/test/resources/security/admin-cert.pem")) + extraConfigFile("node-cert.pem", new File("$rootDir/src/test/resources/security/node-cert.pem")) + extraConfigFile("node-key.pem", new File("$rootDir/src/test/resources/security/node-key.pem")) + extraConfigFile("root-ca.pem", new File("$rootDir/src/test/resources/security/root-ca.pem")) + + setting("plugins.security.ssl.transport.pemcert_filepath", "node-cert.pem") + setting("plugins.security.ssl.transport.pemkey_filepath", "node-key.pem") + setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") + setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") + setting("plugins.security.ssl.http.enabled", "true") + setting("plugins.security.ssl.http.pemcert_filepath", "node-cert.pem") + setting("plugins.security.ssl.http.pemkey_filepath", "node-key.pem") + setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") + setting("plugins.security.allow_unsafe_democertificates", "true") + setting("plugins.security.allow_default_init_securityindex", "true") + //setting("plugins.security.unsupported.inject_user.enabled", "true") + + + setting("plugins.security.authcz.admin_dn", "\n- CN=admin,OU=client,O=client,L=test, C=de") + setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]') + setting('plugins.security.system_indices.enabled', "true") + setting('plugins.security.system_indices.indices', '[".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]') + + setSecure(true) + } + plugin(project.tasks.bundlePlugin.archiveFile) if (Os.isFamily(Os.FAMILY_WINDOWS)) { // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables diff --git a/src/test/resources/security/admin-cert.pem b/src/test/resources/security/admin-cert.pem new file mode 100644 index 0000000000..6c5ede9216 --- /dev/null +++ b/src/test/resources/security/admin-cert.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEmDCCA4CgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iYwDQYJKoZIhvcNAQEL +BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt +cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl +IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v +dCBDQTAeFw0yMzA4MjkyMDA2MzdaFw0zMzA4MjYyMDA2MzdaME0xCzAJBgNVBAYT +AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs +aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs +paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+ +O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx +vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6 +cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0 +bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw +DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMIHPBgNVHSME +gccwgcSAFBeH36Ba62YSp9XQ+LoSRTy3KwCcoYGVpIGSMIGPMRMwEQYKCZImiZPy +LGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhh +bXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENB +MSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCFHfkrz782p+T9k0G +xGeM4+BrehWKMB0GA1UdDgQWBBSjMS8tgguX/V7KSGLoGg7K6XMzIDANBgkqhkiG +9w0BAQsFAAOCAQEANMwD1JYlwAh82yG1gU3WSdh/tb6gqaSzZK7R6I0L7slaXN9m +y2ErUljpTyaHrdiBFmPhU/2Kj2r+fIUXtXdDXzizx/JdmueT0nG9hOixLqzfoC9p +fAhZxM62RgtyZoaczQN82k1/geMSwRpEndFe3OH7arkS/HSbIFxQhAIy229eWe5d +1bUzP59iu7f3r567I4ob8Vy7PP+Ov35p7Vv4oDHHwgsdRzX6pvL6mmwVrQ3BfVec +h9Dqprr+ukYmjho76g6k5cQuRaB6MxqldzUg+2E7IHQP8MCF+co51uZq2nl33mtp +RGr6JbdHXc96zsLTL3saJQ8AWEfu1gbTVrwyRA== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/src/test/resources/security/admin-key.pem b/src/test/resources/security/admin-key.pem new file mode 100644 index 0000000000..1949c26139 --- /dev/null +++ b/src/test/resources/security/admin-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp +gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky +AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo +7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB +GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+ +b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu +y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4 +ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0 +TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j +xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ +OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo +1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs +9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs +/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3 +qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG +/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv +M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0 +0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ +K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5 +9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF +RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp +nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5 +3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h +mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw +F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs +/AHmo368d4PSNRMMzLHw8Q== +-----END PRIVATE KEY----- \ No newline at end of file diff --git a/src/test/resources/security/node-cert.pem b/src/test/resources/security/node-cert.pem new file mode 100644 index 0000000000..671f945396 --- /dev/null +++ b/src/test/resources/security/node-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL +BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt +cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl +IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v +dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT +AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl +MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud +yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0 +HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr +XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n +dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD +ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R +BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA +AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF +BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo +wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ +KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR +MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27 +zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N +1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy +vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L +zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/src/test/resources/security/node-key.pem b/src/test/resources/security/node-key.pem new file mode 100644 index 0000000000..e90562be43 --- /dev/null +++ b/src/test/resources/security/node-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCm93kXteDQHMAv +bUPNPW5pyRHKDD42XGWSgq0k1D29C/UdyL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0 +o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0HGkn47XVu3EwbfrTENg3jFu+Oem6a/50 +1SzITzJWtS0cn2dIFOBimTVpT/4Zv5qrXA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1 +MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8ndibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b +6l+KLo3IKpfTbAIJXIO+M67FLtWKtttDao94B069skzKk6FPgW/OZh6PRCD0oxOa +vV+ld2SjAgMBAAECggEAQK1+uAOZeaSZggW2jQut+MaN4JHLi61RH2cFgU3COLgo +FIiNjFn8f2KKU3gpkt1It8PjlmprpYut4wHI7r6UQfuv7ZrmncRiPWHm9PB82+ZQ +5MXYqj4YUxoQJ62Cyz4sM6BobZDrjG6HHGTzuwiKvHHkbsEE9jQ4E5m7yfbVvM0O +zvwrSOM1tkZihKSTpR0j2+taji914tjBssbn12TMZQL5ItGnhR3luY8mEwT9MNkZ +xg0VcREoAH+pu9FE0vPUgLVzhJ3be7qZTTSRqv08bmW+y1plu80GbppePcgYhEow +dlW4l6XPJaHVSn1lSFHE6QAx6sqiAnBz0NoTPIaLyQKBgQDZqDOlhCRciMRicSXn +7yid9rhEmdMkySJHTVFOidFWwlBcp0fGxxn8UNSBcXdSy7GLlUtH41W9PWl8tp9U +hQiiXORxOJ7ZcB80uNKXF01hpPj2DpFPWyHFxpDkWiTAYpZl68rOlYujxZUjJIej +VvcykBC2BlEOG9uZv2kxcqLyJwKBgQDEYULTxaTuLIa17wU3nAhaainKB3vHxw9B +Ksy5p3ND43UNEKkQm7K/WENx0q47TA1mKD9i+BhaLod98mu0YZ+BCUNgWKcBHK8c +uXpauvM/pLhFLXZ2jvEJVpFY3J79FSRK8bwE9RgKfVKMMgEk4zOyZowS8WScOqiy +hnQn1vKTJQKBgElhYuAnl9a2qXcC7KOwRsJS3rcKIVxijzL4xzOyVShp5IwIPbOv +hnxBiBOH/JGmaNpFYBcBdvORE9JfA4KMQ2fx53agfzWRjoPI1/7mdUk5RFI4gRb/ +A3jZRBoopgFSe6ArCbnyQxzYzToG48/Wzwp19ZxYrtUR4UyJct6f5n27AoGBAJDh +KIpQQDOvCdtjcbfrF4aM2DPCfaGPzENJriwxy6oEPzDaX8Bu/dqI5Ykt43i/zQrX +GpyLaHvv4+oZVTiI5UIvcVO9U8hQPyiz9f7F+fu0LHZs6f7hyhYXlbe3XFxeop3f +5dTKdWgXuTTRF2L9dABkA2deS9mutRKwezWBMQk5AoGBALPtX0FrT1zIosibmlud +tu49A/0KZu4PBjrFMYTSEWGNJez3Fb2VsJwylVl6HivwbP61FhlYfyksCzQQFU71 ++x7Nmybp7PmpEBECr3deoZKQ/acNHn0iwb0It+YqV5+TquQebqgwK6WCLsMuiYKT +bg/ch9Rhxbq22yrVgWHh6epp +-----END PRIVATE KEY----- \ No newline at end of file diff --git a/src/test/resources/security/root-ca.pem b/src/test/resources/security/root-ca.pem new file mode 100644 index 0000000000..02b0e6bd77 --- /dev/null +++ b/src/test/resources/security/root-ca.pem @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL +BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt +cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl +IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v +dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm +iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ +RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 +IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEPyN7J9VGPyJcQmCBl5TGwfSzvVdWwoQU +j9aEsdfFJ6pBCDQSsj8Lv4RqL0dZra7h7SpZLLX/YZcnjikrYC+rP5OwsI9xEE/4 +U98CsTBPhIMgqFK6SzNE5494BsAk4cL72dOOc8tX19oDS/PvBULbNkthQ0aAF1dg +vbrHvu7hq7LisB5ZRGHVE1k/AbCs2PaaKkn2jCw/b+U0Ml9qPuuEgz2mAqJDGYoA +WSR4YXrOcrmPuRqbws464YZbJW898/0Pn/U300ed+4YHiNYLLJp51AMkR4YEw969 +VRPbWIvLrd0PQBooC/eLrL6rvud/GpYhdQEUx8qcNCKd4bz3OaQ5AgMBAAGjggEW +MIIBEjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQU +F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4 +uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ +k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD +VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg +Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN +AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f +qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i +jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD +jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae +dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du +8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y= +-----END CERTIFICATE----- \ No newline at end of file From 7887485cac50c329cb8c3435818d8b2b97684dbe Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Thu, 28 Dec 2023 17:14:55 -0800 Subject: [PATCH 2/8] Further improvements to getting security working Signed-off-by: John Mazanec --- .github/workflows/CI.yml | 40 ------ .github/workflows/test_security.yml | 92 +++--------- build.gradle | 131 +++++++++-------- src/test/resources/security/admin-cert.pem | 27 ---- src/test/resources/security/admin-key.pem | 28 ---- src/test/resources/security/node-cert.pem | 25 ---- src/test/resources/security/node-key.pem | 28 ---- src/test/resources/security/root-ca.pem | 28 ---- .../org/opensearch/knn/ODFERestTestCase.java | 132 ++++-------------- 9 files changed, 115 insertions(+), 416 deletions(-) delete mode 100644 src/test/resources/security/admin-cert.pem delete mode 100644 src/test/resources/security/admin-key.pem delete mode 100644 src/test/resources/security/node-cert.pem delete mode 100644 src/test/resources/security/node-key.pem delete mode 100644 src/test/resources/security/root-ca.pem diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c807e348e6..b58741b921 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -125,43 +125,3 @@ jobs: run: | ./gradlew.bat build -# - name: Pull and Run Docker for security tests -# run: | -# plugin=`ls build/distributions/*.zip` -# version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3` -# plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4` -# echo $version -# cd .. -# if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version -# then -# echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile -# echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile -# echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile -# echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile -# echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile -# echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile -# echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile -# echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile -# echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile -# docker build -t odfe-knn:test . -# echo "imagePresent=true" >> $GITHUB_ENV -# else -# echo "imagePresent=false" >> $GITHUB_ENV -# fi -# - name: Run Docker Image -# if: env.imagePresent == 'true' -# run: | -# cd .. -# docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test -# sleep 90 -# - name: Run k-NN Test -# if: env.imagePresent == 'true' -# run: | -# security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l` -# if [ $security -gt 0 ] -# then -# echo "Security plugin is available. Running tests in security mode" -# ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin -# else -# echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build" -# fi diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index ff3ca14595..85a9a0fefc 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -12,87 +12,39 @@ on: - "feature/**" jobs: - Build-ad: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + integ-test-with-security-linux: strategy: matrix: - java: [ 11,17,21 ] - os: [ubuntu-latest] - fail-fast: true + java: [11, 17, 21] - name: Test k-NN on Secure Cluster - runs-on: ${{ matrix.os }} + name: Build and Test k-NN Plugin on Linux + runs-on: ubuntu-latest + needs: Get-CI-Image-Tag + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Checkout k-NN uses: actions/checkout@v1 + with: + submodules: true - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - name: Install dependencies on ubuntu - if: startsWith(matrix.os,'ubuntu') - run: | - sudo apt-get install libopenblas-dev gfortran -y - - - name: Assemble k-NN - run: | - ./gradlew assemble - # example of variables: - # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip - # version = 2.7.0 - # plugin_version = 2.7.0.0 - # qualifier = `SNAPSHOT` - - name: Pull and Run Docker - run: | - plugin=`basename $(ls build/distributions/*.zip)` - version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` - plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` - qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` - if [ $qualifier != `SNAPSHOT` ]; - then - docker_version=$version-$qualifier - else - docker_version=$version - fi - echo plugin version plugin_version qualifier docker_version - echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" - - cd .. - if docker pull opensearchstaging/opensearch:$docker_version - then - echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile - # knn plugin cannot be deleted until there are plugin that has dependency on it - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile - # saving pre-built artifacts of native libraries as we can't build it with gradle assemle - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile - echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile - echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile - # moving pre-built artifacts of native libraries back to plugin folder - echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile - docker build -t opensearch-knn:test . - echo "imagePresent=true" >> $GITHUB_ENV - else - echo "imagePresent=false" >> $GITHUB_ENV - fi - - - name: Run Docker Image - if: env.imagePresent == 'true' - run: | - cd .. - docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test - sleep 90 - - name: Run k-NN Integ Test - if: env.imagePresent == 'true' + - name: Run build + # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | - security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l` - if [ $security -gt 0 ] - then - echo "Security plugin is available" - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin - else - echo "Security plugin is NOT available, skipping integration tests" - fi + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true" diff --git a/build.gradle b/build.gradle index 1ac273ee6f..d41deac7ab 100644 --- a/build.gradle +++ b/build.gradle @@ -4,8 +4,9 @@ */ import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.OpenSearchCluster import org.apache.tools.ant.taskdefs.condition.Os - +import java.nio.file.Paths import java.util.concurrent.Callable buildscript { @@ -58,6 +59,7 @@ plugins { id 'idea' id "com.diffplug.spotless" version "6.20.0" apply false id 'io.freefair.lombok' version '8.4' + id "de.undercouch.download" version "5.3.0" } apply from: 'gradle/formatting.gradle' @@ -66,8 +68,66 @@ apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.pluginzip' apply plugin: 'opensearch.repositories' + +def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile +opensearch_tmp_dir.mkdirs() + ext { projectSubstitutions = [:] + + configureSecurityPlugin = { OpenSearchCluster cluster -> + configurations.zipArchive.asFileTree.each { + cluster.plugin(provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it + } + } + } + })) + } + + cluster.getNodes().forEach { node -> + var creds = node.getCredentials() + if (creds.isEmpty()) { + creds.add(Map.of('username', 'admin', 'password', 'admin')) + } else { + creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin')) + } + } + + // Config below including files are copied from security demo configuration + ['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file -> + File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile() + download.run { + src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file + dest local + overwrite false + } + cluster.extraConfigFile(file, local) + } + + cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") + cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") + cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") + cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") + cluster.setting("plugins.security.ssl.http.enabled", "true") + cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem") + cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem") + cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") + cluster.setting("plugins.security.allow_unsafe_democertificates", "true") + cluster.setting("plugins.security.allow_default_init_securityindex", "true") + cluster.setting("plugins.security.unsupported.inject_user.enabled", "true") + + cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de") + cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]') + cluster.setting('plugins.security.system_indices.enabled', "true") + cluster.setting('plugins.security.system_indices.indices', '[".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]') + cluster.setSecure(true) + } } allprojects { @@ -204,10 +264,6 @@ dependencies { zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}" } - -def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile -opensearch_tmp_dir.mkdirs() - task windowsPatches(type:Exec) { commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1" } @@ -252,13 +308,13 @@ integTest { // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) - systemProperty("tests.opensearch.https", "true") - systemProperty("tests.opensearch.username", "admin") - systemProperty("tests.opensearch.password", "admin") - -// systemProperty "https", System.getProperty("https") -// systemProperty "user", System.getProperty("user") -// systemProperty "password", System.getProperty("password") + if (System.getProperty("security.enabled") != null) { + // These need to be set for configuring the rest client + systemProperty("https", "true") + systemProperty("secure", "true") + systemProperty("user", "admin") + systemProperty("password", "admin") + } doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can @@ -269,10 +325,6 @@ integTest { systemProperty 'cluster.number_of_nodes', "${_numNodes}" // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written - - - - getClusters().forEach { cluster -> cluster.waitForAllConditions() } @@ -289,52 +341,7 @@ testClusters.integTest { // Optionally install security if (System.getProperty("security.enabled") != null) { - configurations.zipArchive.asFileTree.each { - plugin(provider(new Callable() { - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return it - } - } - } - })) - } - -// systemProperty("tests.opensearch.https", "true") -// systemProperty("tests.opensearch.username", "admin") -// systemProperty("tests.opensearch.password", "admin") - - systemProperty("secure", "true") - systemProperty("username", "admin") - systemProperty("password", "admin") - - extraConfigFile("admin-cert.pem", new File("$rootDir/src/test/resources/security/admin-cert.pem")) - extraConfigFile("node-cert.pem", new File("$rootDir/src/test/resources/security/node-cert.pem")) - extraConfigFile("node-key.pem", new File("$rootDir/src/test/resources/security/node-key.pem")) - extraConfigFile("root-ca.pem", new File("$rootDir/src/test/resources/security/root-ca.pem")) - - setting("plugins.security.ssl.transport.pemcert_filepath", "node-cert.pem") - setting("plugins.security.ssl.transport.pemkey_filepath", "node-key.pem") - setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") - setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") - setting("plugins.security.ssl.http.enabled", "true") - setting("plugins.security.ssl.http.pemcert_filepath", "node-cert.pem") - setting("plugins.security.ssl.http.pemkey_filepath", "node-key.pem") - setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") - setting("plugins.security.allow_unsafe_democertificates", "true") - setting("plugins.security.allow_default_init_securityindex", "true") - //setting("plugins.security.unsupported.inject_user.enabled", "true") - - - setting("plugins.security.authcz.admin_dn", "\n- CN=admin,OU=client,O=client,L=test, C=de") - setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]') - setting('plugins.security.system_indices.enabled', "true") - setting('plugins.security.system_indices.indices', '[".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]') - - setSecure(true) + configureSecurityPlugin(testClusters.integTest) } plugin(project.tasks.bundlePlugin.archiveFile) diff --git a/src/test/resources/security/admin-cert.pem b/src/test/resources/security/admin-cert.pem deleted file mode 100644 index 6c5ede9216..0000000000 --- a/src/test/resources/security/admin-cert.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEmDCCA4CgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iYwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkyMDA2MzdaFw0zMzA4MjYyMDA2MzdaME0xCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs -aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs -paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+ -O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx -vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6 -cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0 -bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw -DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMIHPBgNVHSME -gccwgcSAFBeH36Ba62YSp9XQ+LoSRTy3KwCcoYGVpIGSMIGPMRMwEQYKCZImiZPy -LGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhh -bXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENB -MSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCFHfkrz782p+T9k0G -xGeM4+BrehWKMB0GA1UdDgQWBBSjMS8tgguX/V7KSGLoGg7K6XMzIDANBgkqhkiG -9w0BAQsFAAOCAQEANMwD1JYlwAh82yG1gU3WSdh/tb6gqaSzZK7R6I0L7slaXN9m -y2ErUljpTyaHrdiBFmPhU/2Kj2r+fIUXtXdDXzizx/JdmueT0nG9hOixLqzfoC9p -fAhZxM62RgtyZoaczQN82k1/geMSwRpEndFe3OH7arkS/HSbIFxQhAIy229eWe5d -1bUzP59iu7f3r567I4ob8Vy7PP+Ov35p7Vv4oDHHwgsdRzX6pvL6mmwVrQ3BfVec -h9Dqprr+ukYmjho76g6k5cQuRaB6MxqldzUg+2E7IHQP8MCF+co51uZq2nl33mtp -RGr6JbdHXc96zsLTL3saJQ8AWEfu1gbTVrwyRA== ------END CERTIFICATE----- \ No newline at end of file diff --git a/src/test/resources/security/admin-key.pem b/src/test/resources/security/admin-key.pem deleted file mode 100644 index 1949c26139..0000000000 --- a/src/test/resources/security/admin-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp -gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky -AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo -7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB -GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+ -b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu -y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4 -ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0 -TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j -xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ -OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo -1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs -9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs -/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3 -qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG -/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv -M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0 -0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ -K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5 -9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF -RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp -nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5 -3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h -mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw -F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs -/AHmo368d4PSNRMMzLHw8Q== ------END PRIVATE KEY----- \ No newline at end of file diff --git a/src/test/resources/security/node-cert.pem b/src/test/resources/security/node-cert.pem deleted file mode 100644 index 671f945396..0000000000 --- a/src/test/resources/security/node-cert.pem +++ /dev/null @@ -1,25 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl -MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud -yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0 -HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr -XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n -dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD -ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R -BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA -AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF -BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo -wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ -KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR -MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27 -zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N -1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy -vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L -zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo= ------END CERTIFICATE----- \ No newline at end of file diff --git a/src/test/resources/security/node-key.pem b/src/test/resources/security/node-key.pem deleted file mode 100644 index e90562be43..0000000000 --- a/src/test/resources/security/node-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCm93kXteDQHMAv -bUPNPW5pyRHKDD42XGWSgq0k1D29C/UdyL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0 -o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0HGkn47XVu3EwbfrTENg3jFu+Oem6a/50 -1SzITzJWtS0cn2dIFOBimTVpT/4Zv5qrXA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1 -MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8ndibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b -6l+KLo3IKpfTbAIJXIO+M67FLtWKtttDao94B069skzKk6FPgW/OZh6PRCD0oxOa -vV+ld2SjAgMBAAECggEAQK1+uAOZeaSZggW2jQut+MaN4JHLi61RH2cFgU3COLgo -FIiNjFn8f2KKU3gpkt1It8PjlmprpYut4wHI7r6UQfuv7ZrmncRiPWHm9PB82+ZQ -5MXYqj4YUxoQJ62Cyz4sM6BobZDrjG6HHGTzuwiKvHHkbsEE9jQ4E5m7yfbVvM0O -zvwrSOM1tkZihKSTpR0j2+taji914tjBssbn12TMZQL5ItGnhR3luY8mEwT9MNkZ -xg0VcREoAH+pu9FE0vPUgLVzhJ3be7qZTTSRqv08bmW+y1plu80GbppePcgYhEow -dlW4l6XPJaHVSn1lSFHE6QAx6sqiAnBz0NoTPIaLyQKBgQDZqDOlhCRciMRicSXn -7yid9rhEmdMkySJHTVFOidFWwlBcp0fGxxn8UNSBcXdSy7GLlUtH41W9PWl8tp9U -hQiiXORxOJ7ZcB80uNKXF01hpPj2DpFPWyHFxpDkWiTAYpZl68rOlYujxZUjJIej -VvcykBC2BlEOG9uZv2kxcqLyJwKBgQDEYULTxaTuLIa17wU3nAhaainKB3vHxw9B -Ksy5p3ND43UNEKkQm7K/WENx0q47TA1mKD9i+BhaLod98mu0YZ+BCUNgWKcBHK8c -uXpauvM/pLhFLXZ2jvEJVpFY3J79FSRK8bwE9RgKfVKMMgEk4zOyZowS8WScOqiy -hnQn1vKTJQKBgElhYuAnl9a2qXcC7KOwRsJS3rcKIVxijzL4xzOyVShp5IwIPbOv -hnxBiBOH/JGmaNpFYBcBdvORE9JfA4KMQ2fx53agfzWRjoPI1/7mdUk5RFI4gRb/ -A3jZRBoopgFSe6ArCbnyQxzYzToG48/Wzwp19ZxYrtUR4UyJct6f5n27AoGBAJDh -KIpQQDOvCdtjcbfrF4aM2DPCfaGPzENJriwxy6oEPzDaX8Bu/dqI5Ykt43i/zQrX -GpyLaHvv4+oZVTiI5UIvcVO9U8hQPyiz9f7F+fu0LHZs6f7hyhYXlbe3XFxeop3f -5dTKdWgXuTTRF2L9dABkA2deS9mutRKwezWBMQk5AoGBALPtX0FrT1zIosibmlud -tu49A/0KZu4PBjrFMYTSEWGNJez3Fb2VsJwylVl6HivwbP61FhlYfyksCzQQFU71 -+x7Nmybp7PmpEBECr3deoZKQ/acNHn0iwb0It+YqV5+TquQebqgwK6WCLsMuiYKT -bg/ch9Rhxbq22yrVgWHh6epp ------END PRIVATE KEY----- \ No newline at end of file diff --git a/src/test/resources/security/root-ca.pem b/src/test/resources/security/root-ca.pem deleted file mode 100644 index 02b0e6bd77..0000000000 --- a/src/test/resources/security/root-ca.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm -iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ -RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 -IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEPyN7J9VGPyJcQmCBl5TGwfSzvVdWwoQU -j9aEsdfFJ6pBCDQSsj8Lv4RqL0dZra7h7SpZLLX/YZcnjikrYC+rP5OwsI9xEE/4 -U98CsTBPhIMgqFK6SzNE5494BsAk4cL72dOOc8tX19oDS/PvBULbNkthQ0aAF1dg -vbrHvu7hq7LisB5ZRGHVE1k/AbCs2PaaKkn2jCw/b+U0Ml9qPuuEgz2mAqJDGYoA -WSR4YXrOcrmPuRqbws464YZbJW898/0Pn/U300ed+4YHiNYLLJp51AMkR4YEw969 -VRPbWIvLrd0PQBooC/eLrL6rvud/GpYhdQEUx8qcNCKd4bz3OaQ5AgMBAAGjggEW -MIIBEjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQU -F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4 -uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ -k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD -VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg -Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN -AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f -qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i -jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD -jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae -dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du -8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y= ------END CERTIFICATE----- \ No newline at end of file diff --git a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java index 0b951e5c39..887951d2b7 100644 --- a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java @@ -6,14 +6,10 @@ package org.opensearch.knn; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -39,14 +35,11 @@ import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; -import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.MediaType; @@ -55,10 +48,7 @@ import org.opensearch.search.SearchHit; import org.opensearch.test.rest.OpenSearchRestTestCase; import org.junit.After; -import org.opensearch.commons.rest.SecureRestClientBuilder; -import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; -import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX; import static org.opensearch.knn.TestUtils.OPENDISTRO_SECURITY; import static org.opensearch.knn.TestUtils.OPENSEARCH_SYSTEM_INDEX_PREFIX; @@ -66,11 +56,6 @@ import static org.opensearch.knn.TestUtils.SKIP_DELETE_MODEL_INDEX; import static org.opensearch.knn.common.KNNConstants.MODELS; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_ENABLED; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH; /** * ODFE integration test base class to support both security disabled and enabled ODFE cluster. @@ -80,15 +65,7 @@ public abstract class ODFERestTestCase extends OpenSearchRestTestCase { private final Set IMMUTABLE_INDEX_PREFIXES = Set.of(KNN_BWC_PREFIX, SECURITY_AUDITLOG_PREFIX, OPENSEARCH_SYSTEM_INDEX_PREFIX); protected boolean isHttps() { - boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); - if (isHttps) { - // currently only external cluster is supported for security enabled testing - if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { - throw new RuntimeException("cluster url should be provided for security enabled testing"); - } - } - - return isHttps; + return Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); } @Override @@ -100,37 +77,19 @@ protected String getProtocol() { protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); if (isHttps()) { - String keystore = settings.get(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH); - if (Objects.nonNull(keystore)) { - URI uri; - try { - uri = this.getClass().getClassLoader().getResource("security/sample.pem").toURI(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - Path configPath = PathUtils.get(uri).getParent().toAbsolutePath(); - return new SecureRestClientBuilder(settings, configPath).build(); - } else { - configureHttpsClient(builder, settings); - boolean strictDeprecationMode = settings.getAsBoolean("strictDeprecationMode", true); - builder.setStrictDeprecationMode(strictDeprecationMode); - return builder.build(); - } + configureHttpsClient(builder, settings); } else { configureClient(builder, settings); } + builder.setStrictDeprecationMode(false); return builder.build(); } protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { - Map headers = ThreadContext.buildDefaultHeaders(settings); - Header[] defaultHeaders = new Header[headers.size()]; - int i = 0; - for (Map.Entry entry : headers.entrySet()) { - defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); - } - builder.setDefaultHeaders(defaultHeaders); + // Similar to client configuration with OpenSearch: + // https://github.com/opensearch-project/OpenSearch/blob/2.11.1/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java#L841-L863 + // except we set the user name and password builder.setHttpClientConfigCallback(httpClientBuilder -> { String userName = Optional.ofNullable(System.getProperty("user")) .orElseThrow(() -> new RuntimeException("user name is missing")); @@ -147,8 +106,6 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s .setTlsDetailsFactory(sslEngine -> new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol())) .build(); final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() - .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE) - .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL) .setTlsStrategy(tlsStrategy) .build(); return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager); @@ -156,18 +113,21 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s throw new RuntimeException(e); } }); - + Map headers = ThreadContext.buildDefaultHeaders(settings); + Header[] defaultHeaders = new Header[headers.size()]; + int i = 0; + for (Map.Entry entry : headers.entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); + } + builder.setDefaultHeaders(defaultHeaders); final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); final TimeValue socketTimeout = TimeValue.parseTimeValue( socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT ); - builder.setRequestConfigCallback(conf -> { - Timeout timeout = Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis())); - conf.setConnectTimeout(timeout); - conf.setResponseTimeout(timeout); - return conf; - }); + builder.setRequestConfigCallback( + conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis()))) + ); if (settings.hasValue(CLIENT_PATH_PREFIX)) { builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); } @@ -204,8 +164,10 @@ protected void wipeAllODFEIndices() throws Exception { for (Map index : parserList) { final String indexName = (String) index.get("index"); - if (isIndexCleanupRequired(indexName)) { - wipeIndexContent(indexName); + if (MODEL_INDEX_NAME.equals(indexName)) { + if (!getSkipDeleteModelIndexFlag()) { + deleteModels(getModelIds()); + } continue; } if (!skipDeleteIndex(indexName)) { @@ -215,15 +177,6 @@ protected void wipeAllODFEIndices() throws Exception { } } - private boolean isIndexCleanupRequired(final String index) { - return MODEL_INDEX_NAME.equals(index) && !getSkipDeleteModelIndexFlag(); - } - - private void wipeIndexContent(String indexName) throws IOException, ParseException { - deleteModels(getModelIds()); - deleteAllDocs(indexName); - } - private List getModelIds() throws IOException, ParseException { final String restURIGetModels = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); final Response response = adminClient().performRequest(new Request("GET", restURIGetModels)); @@ -251,51 +204,14 @@ private void deleteModels(final List modelIds) throws IOException { } } - private void deleteAllDocs(final String indexName) throws IOException { - final String restURIDeleteByQuery = String.join("/", indexName, "_delete_by_query"); - final Request request = new Request("POST", restURIDeleteByQuery); - final XContentBuilder matchAllDocsQuery = XContentFactory.jsonBuilder() - .startObject() - .startObject("query") - .startObject("match_all") - .endObject() - .endObject() - .endObject(); - - request.setJsonEntity(matchAllDocsQuery.toString()); - adminClient().performRequest(request); - } - private boolean getSkipDeleteModelIndexFlag() { return Boolean.parseBoolean(System.getProperty(SKIP_DELETE_MODEL_INDEX, "false")); } - private boolean skipDeleteModelIndex(String indexName) { - return (MODEL_INDEX_NAME.equals(indexName) && getSkipDeleteModelIndexFlag()); - } - private boolean skipDeleteIndex(String indexName) { - if (indexName != null - && !OPENDISTRO_SECURITY.equals(indexName) - && IMMUTABLE_INDEX_PREFIXES.stream().noneMatch(indexName::startsWith) - && !skipDeleteModelIndex(indexName)) { - return false; - } - - return true; - } - - @Override - protected Settings restAdminSettings() { - return Settings.builder() - // disable the warning exception for admin client since it's only used for cleanup. - .put("strictDeprecationMode", false) - .put("http.port", 9200) - .put(OPENSEARCH_SECURITY_SSL_HTTP_ENABLED, isHttps()) - .put(OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH, "sample.pem") - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, "test-kirk.jks") - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD, "changeit") - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD, "changeit") - .build(); + return indexName == null + || OPENDISTRO_SECURITY.equals(indexName) + || IMMUTABLE_INDEX_PREFIXES.stream().anyMatch(indexName::startsWith) + || MODEL_INDEX_NAME.equals(indexName); } } From 1a54cf8911dff7673e4b9e298c814ed54ea33f13 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 29 Dec 2023 09:16:04 -0800 Subject: [PATCH 3/8] Clean up and add docs Signed-off-by: John Mazanec --- .github/workflows/test_security.yml | 2 +- CHANGELOG.md | 2 +- DEVELOPER_GUIDE.md | 37 +++++++++++++++++++++++++- build.gradle | 40 ++++++++++++++++++++++++----- 4 files changed, 72 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 85a9a0fefc..4d0374a081 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -22,7 +22,7 @@ jobs: matrix: java: [11, 17, 21] - name: Build and Test k-NN Plugin on Linux + name: Run Integration Tests on Linux runs-on: ubuntu-latest needs: Get-CI-Image-Tag container: diff --git a/CHANGELOG.md b/CHANGELOG.md index bf61d49e7b..4fb5a86eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,9 +22,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) * Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) - ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) +* Refactor security testing to install from individual components [#1317](https://github.com/opensearch-project/k-NN/pull/1307) ### Documentation ### Maintenance * Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index cbfaf0f4c6..196fe91bb1 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -264,6 +264,36 @@ curl localhost:9200 } } ``` + +Additionally, it is also possible to run a cluster with security enabled: +```shell script +./gradlew run -Dsecurity.enabled=true -Dhttps=true -Duser=admin -Dpassword=admin +``` + +By default, if `-Dsecurity.enabled=true` is passed the following defaults will be used: `https=true`, `user=admin` and +`password=admin`. + +Then, to access the cluster, we can run +```bash +curl https://localhost:9200 --insecure -u admin:admin + +{ + "name" : "runTask-0", + "cluster_name" : "runTask", + "cluster_uuid" : "oX_S6cxGSgOr_mNnUxO6yQ", + "version" : { + "number" : "1.0.0-SNAPSHOT", + "build_type" : "tar", + "build_hash" : "0ba0e7cc26060f964fcbf6ee45bae53b3a9941d0", + "build_date" : "2021-04-16T19:45:44.248303Z", + "build_snapshot" : true, + "lucene_version" : "8.7.0", + "minimum_wire_compatibility_version" : "6.8.0", + "minimum_index_compatibility_version" : "6.0.0-beta1" + } +} +``` + ### Run Multi-node Cluster Locally It can be useful to test and debug on a multi-node cluster. In order to launch a 3 node cluster with the KNN plugin installed, run the following command: @@ -272,12 +302,17 @@ It can be useful to test and debug on a multi-node cluster. In order to launch a ./gradlew run -PnumNodes=3 ``` -In order to run the integration tests with a 3 node cluster, run this command: +In order to run the integration tests, run this command: ``` ./gradlew :integTest -PnumNodes=3 ``` +Additionally, to run integration tests with security enabled, run +``` +./gradlew :integTest -Dsecurity.enabled=true -PnumNodes=3 +``` + Integration tests can be run with remote cluster. For that run the following command and replace host/port/cluster name values with ones for the target cluster: ``` diff --git a/build.gradle b/build.gradle index d41deac7ab..679262b1f9 100644 --- a/build.gradle +++ b/build.gradle @@ -125,7 +125,30 @@ ext { cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de") cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]') cluster.setting('plugins.security.system_indices.enabled', "true") - cluster.setting('plugins.security.system_indices.indices', '[".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]') + cluster.setting('plugins.security.system_indices.indices', '[' + + '".plugins-ml-config", ' + + '".plugins-ml-connector", ' + + '".plugins-ml-model-group", ' + + '".plugins-ml-model", ".plugins-ml-task", ' + + '".plugins-ml-conversation-meta", ' + + '".plugins-ml-conversation-interactions", ' + + '".opendistro-alerting-config", ' + + '".opendistro-alerting-alert*", ' + + '".opendistro-anomaly-results*", ' + + '".opendistro-anomaly-detector*", ' + + '".opendistro-anomaly-checkpoints", ' + + '".opendistro-anomaly-detection-state", ' + + '".opendistro-reports-*", ' + + '".opensearch-notifications-*", ' + + '".opensearch-notebooks", ' + + '".opensearch-observability", ' + + '".ql-datasources", ' + + '".opendistro-asynchronous-search-response*", ' + + '".replication-metadata-store", ' + + '".opensearch-knn-models", ' + + '".geospatial-ip2geo-data*"' + + ']' + ) cluster.setSecure(true) } } @@ -308,13 +331,18 @@ integTest { // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) + var is_https = System.getProperty("https") + var user = System.getProperty("user") + var password = System.getProperty("password") if (System.getProperty("security.enabled") != null) { - // These need to be set for configuring the rest client - systemProperty("https", "true") - systemProperty("secure", "true") - systemProperty("user", "admin") - systemProperty("password", "admin") + // If security is enabled, set is_https/user/password defaults + is_https = is_https == null ? "true" : is_https + user = user == null ? "admin" : user + password = password == null ? "admin" : password } + systemProperty("https", is_https) + systemProperty("user", user) + systemProperty("password", password) doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can From db931fef4c57fe365883f01a4da0ce07fb4d81ae Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 29 Dec 2023 10:21:07 -0800 Subject: [PATCH 4/8] Remove test cases that directly update model index Signed-off-by: John Mazanec --- .../java/org/opensearch/knn/bwc/ModelIT.java | 28 +-------------- .../org/opensearch/knn/KNNRestTestCase.java | 34 ------------------- 2 files changed, 1 insertion(+), 61 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index 7eb75e24c6..dc9219adcf 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -10,21 +10,16 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.client.ResponseException; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; @@ -55,7 +50,7 @@ public class ModelIT extends AbstractRestartUpgradeTestCase { private static int DOC_ID_TEST_MODEL_INDEX = 0; private static int DOC_ID_TEST_MODEL_INDEX_DEFAULT = 0; private static final int DELAY_MILLI_SEC = 1000; - private static final int EXP_NUM_OF_MODELS = 3; + private static final int EXP_NUM_OF_MODELS = 2; private static final int K = 5; private static final int NUM_DOCS = 10; private static final int NUM_DOCS_TEST_MODEL_INDEX = 100; @@ -66,7 +61,6 @@ public class ModelIT extends AbstractRestartUpgradeTestCase { private static int QUERY_COUNT_TEST_MODEL_INDEX_DEFAULT = 0; private static final String TEST_MODEL_ID = "test-model-id"; private static final String TEST_MODEL_ID_DEFAULT = "test-model-id-default"; - private static final String TEST_MODEL_ID_TRAINING = "test-model-id-training"; private static final String MODEL_DESCRIPTION = "Description for train model test"; // KNN model test @@ -139,22 +133,6 @@ public void testKNNModelDefault() throws Exception { } } - // KNN Delete Model test for model in Training State - public void testDeleteTrainingModel() throws Exception { - byte[] testModelBlob = "hello".getBytes(StandardCharsets.UTF_8); - ModelMetadata testModelMetadata = getModelMetadata(); - testModelMetadata.setState(ModelState.TRAINING); - if (isRunningAgainstOldCluster()) { - addModelToSystemIndex(TEST_MODEL_ID_TRAINING, testModelMetadata, testModelBlob); - } else { - String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, TEST_MODEL_ID_TRAINING); - Request request = new Request("DELETE", restURI); - - ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); - assertEquals(RestStatus.CONFLICT.getStatus(), ex.getResponse().getStatusLine().getStatusCode()); - } - } - // Delete Models and ".opensearch-knn-models" index to clear cluster metadata @AfterClass public static void wipeAllModels() throws IOException { @@ -255,8 +233,4 @@ public String modelIndexMapping(String fieldName, String modelId) throws IOExcep .endObject() .toString(); } - - private ModelMetadata getModelMetadata() { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", "", ""); - } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index d68d2b8d86..56e63d53c9 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -22,7 +22,6 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.script.KNNScoringScriptEngine; @@ -56,7 +55,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -76,15 +74,10 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.MODEL_BLOB_PARAMETER; import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; -import static org.opensearch.knn.common.KNNConstants.MODEL_NODE_ASSIGNMENT; -import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_MAPPING_PATH; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; -import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; import static org.opensearch.knn.common.KNNConstants.NAME; @@ -764,33 +757,6 @@ protected void createModelSystemIndex() throws IOException { } } - protected void addModelToSystemIndex(String modelId, ModelMetadata modelMetadata, byte[] model) throws IOException { - assertFalse(org.opensearch.core.common.Strings.isNullOrEmpty(modelId)); - String modelBase64 = Base64.getEncoder().encodeToString(model); - - Request request = new Request("POST", "/" + MODEL_INDEX_NAME + "/_doc/" + modelId + "?refresh=true"); - - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .field(MODEL_ID, modelId) - .field(MODEL_STATE, modelMetadata.getState().getName()) - .field(KNN_ENGINE, modelMetadata.getKnnEngine().getName()) - .field(METHOD_PARAMETER_SPACE_TYPE, modelMetadata.getSpaceType().getValue()) - .field(DIMENSION, modelMetadata.getDimension()) - .field(MODEL_BLOB_PARAMETER, modelBase64) - .field(MODEL_TIMESTAMP, modelMetadata.getTimestamp()) - .field(MODEL_DESCRIPTION, modelMetadata.getDescription()) - .field(MODEL_ERROR, modelMetadata.getError()) - .field(MODEL_NODE_ASSIGNMENT, modelMetadata.getNodeAssignment()) - .endObject(); - - request.setJsonEntity(builder.toString()); - - Response response = client().performRequest(request); - - assertEquals(request.getEndpoint() + ": failed", RestStatus.CREATED, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - } - /** * Clear cache *

From 4e7157e30db03ce1b1d48e267bd83710874a286c Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 29 Dec 2023 13:43:48 -0800 Subject: [PATCH 5/8] Address comments Signed-off-by: John Mazanec --- CHANGELOG.md | 2 +- DEVELOPER_GUIDE.md | 22 ++++++++++++---------- build.gradle | 2 ++ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fb5a86eaa..7839d98254 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) -* Refactor security testing to install from individual components [#1317](https://github.com/opensearch-project/k-NN/pull/1307) +* Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) ### Documentation ### Maintenance * Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 196fe91bb1..2e7f7322c8 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -278,19 +278,21 @@ Then, to access the cluster, we can run curl https://localhost:9200 --insecure -u admin:admin { - "name" : "runTask-0", - "cluster_name" : "runTask", - "cluster_uuid" : "oX_S6cxGSgOr_mNnUxO6yQ", + "name" : "integTest-0", + "cluster_name" : "integTest", + "cluster_uuid" : "kLsNk4JDTMyp1yQRqog-3g", "version" : { - "number" : "1.0.0-SNAPSHOT", + "distribution" : "opensearch", + "number" : "3.0.0-SNAPSHOT", "build_type" : "tar", - "build_hash" : "0ba0e7cc26060f964fcbf6ee45bae53b3a9941d0", - "build_date" : "2021-04-16T19:45:44.248303Z", + "build_hash" : "9d85e566894ef53e5f2093618b3d455e4d0a04ce", + "build_date" : "2023-10-30T18:34:06.996519Z", "build_snapshot" : true, - "lucene_version" : "8.7.0", - "minimum_wire_compatibility_version" : "6.8.0", - "minimum_index_compatibility_version" : "6.0.0-beta1" - } + "lucene_version" : "9.8.0", + "minimum_wire_compatibility_version" : "2.12.0", + "minimum_index_compatibility_version" : "2.0.0" + }, + "tagline" : "The OpenSearch Project: https://opensearch.org/" } ``` diff --git a/build.gradle b/build.gradle index 679262b1f9..a0dae25590 100644 --- a/build.gradle +++ b/build.gradle @@ -110,6 +110,8 @@ ext { cluster.extraConfigFile(file, local) } + // This configuration is copied from the security plugins demo install: + // https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388 cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") From 0d6222ca28089295b00c9ef0dedb27e90b784297 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Tue, 2 Jan 2024 13:42:03 -0800 Subject: [PATCH 6/8] Add ensure green to single node test Signed-off-by: John Mazanec --- .../TrainingJobRouteDecisionInfoTransportActionTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java index 9f64afebb4..ccf69b423d 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java @@ -51,6 +51,8 @@ public void teardown() { @SuppressWarnings("unchecked") public void testNodeOperation() throws IOException, InterruptedException, ExecutionException { + ensureGreen(); + // Ensure initial value of train job count is 0 TrainingJobRouteDecisionInfoTransportAction action = node().injector() .getInstance(TrainingJobRouteDecisionInfoTransportAction.class); From 071095b97f19b76899eaf6a711882662b5a6db03 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Tue, 2 Jan 2024 14:08:01 -0800 Subject: [PATCH 7/8] Remove unused keys Signed-off-by: John Mazanec --- src/test/resources/security/sample.pem | 25 ---------------------- src/test/resources/security/test-kirk.jks | Bin 4504 -> 0 bytes 2 files changed, 25 deletions(-) delete mode 100644 src/test/resources/security/sample.pem delete mode 100644 src/test/resources/security/test-kirk.jks diff --git a/src/test/resources/security/sample.pem b/src/test/resources/security/sample.pem deleted file mode 100644 index a1fc20a776..0000000000 --- a/src/test/resources/security/sample.pem +++ /dev/null @@ -1,25 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl -MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud -yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0 -HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr -XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n -dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD -ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R -BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA -AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF -BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo -wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ -KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR -MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27 -zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N -1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy -vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L -zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo= ------END CERTIFICATE----- diff --git a/src/test/resources/security/test-kirk.jks b/src/test/resources/security/test-kirk.jks deleted file mode 100644 index 6dbc51e714784fa58a4209c75deab8b9ed1698ff..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4504 zcma)AXEYp+vt7GZ$?DyT=tPUf>Rt32Rtcg+B4PQKLo)5nT`xBt(f8 zz4zYx{`1az=l47B(|aH0%$a-V&c}OZ28N+d1QLK?7-~f#Qh{)-@KbUEVuBnDwFn`G zTJSH-2g86X{uc$#Cd7a<{=zALBY_C=KPs|Y1i%~&Sotp~4}12H0!$9GfJy&blEDNC z=>%hA9@l)1y-8vD6#cH^U}=KBI0FdeqXH7J!^nt8{(B;j6byi|5|P@4YY{kr2nhrT zsl1TD93_M516EPM#9d4EG(rsFKtBW4^r*(5KwKbTLB){+^0E(}Q+A7HoW0lrA)@i+ zydGtY^95cAh7C?*2qIcESObb&7%#|($|(-eXIiQ#0>bYpj@=?*4?U=5@-ISTdSa4x zOtEjIWb0hr)D^1HVpX7-CjwnsDG8#WM@AVZvyufeW?}`^GtGW7WcGsVl)G*$?lP3S z^GYelg04B!ZBp4GnwCzq@uOLfB4xY#hE;StB61*Yd8?%(Nl9NW{s3+HODy#ik72s%Hj($a8 zhF0>hs}=106=eHlR<&9zT@LuHAUIZWLFWrKQ#$R3^=pv*&-7e6{O_Ji`|s`^^4v@-Hr>`?(V#!ktZ-$-0?Jt1G-G? zE9HvN@-0iPpKSDRsLacPB>#JY4d$KM!zs7xPBvUu4HQ}!Bz$qc)A`=Ver4EBC?!g7b zuW7GvE*puJA=;!bv2_S?8ZQx_n`M?F&kkb{-h zKwO=OA_@auvAUmAsQW~NjYK|}m{>`{*n^45MJ^ph*%K9}8GnxA%-;D^^-}ih8oWP* zXJ#vzJY3e4?&oSey+_=qv19lq zeLI>%Gjx=y!qVzf%Y&c7dgkjEw?^rl8^KxGs^%{Fd_(b51&l(wYCO&Rc~ZUl5^~y> zc}BJ!4+n2KaS|<{vd#M44my1W|M0Y-gfk9<&l%IBje@31-Sr1Mt!fvT(Pe+Gt$Bz? z_up@HJf$b!)YfI|4{%l^JDxgWvp75|nMzg7E)(qZ%=alvt zXMfZg7Z=_eanGP?tBXFKyvFRu$?uMAzg|k-(32orZccxnHGr$(gM%4Hgc&3blJCi; z6j@^Y3XVg*doBz7pms~Jn7 z9>1&oI7bPBOnn7vyV1x>YahPMDy_bySw!71ij);ebzBEUSZK&o1y43I-AuJKXJ~C3 z{ScF0neCZB8?5r>Px#3V%} zq$OY&i2FZH#6&q5i2Yy421o$-o6P@Z2>vgd4p$sB)+@I7CAQvk>m=OVG#EC`^#8Hx zXo}&oS5+Eg(sw4>QN4_Cy_0U!W9o!pxS@}|4s+L{ow)59*P>fYuDV~JqCwTL5s{)3(v zzbM`$E?)E;`zu*Kjpah> zgQl1ucOJOd1|%MDBk_Lsu64*-#r>9orWT19xT!DnCoNv_AnWczl?5a3@Sd4mtPrx@ z;QPqXK#%ve%3=_Sa$)(zJ)mvCYW0$Uim6bQ!S}#H@uPFY+qvmT_x`cr%&q*~6sufG zKKVZ8ebd?WhVYT)or=?jzV*~PLH&t?CH^KO=IX%=oHNr75%vVz=nN9ipHOrX*7{h! zNkaI3@a@JfTINcbD<@;DNwqa&=S5v4pM=tBEMN8HU3}euq?(dEFWfNC>H+2C+1dBA zFs|s&27315cK^vG`LRKX~{Ugw!|2K~TP_VAqXtzNY6)j={rQ zv73v$!psb1ph9o6`kKlGjC8GEdFX9+@{I}q{33}%?v>$a-cw6HGOOLVnv3ITN_D~k zo^QL%)6K#_{j)b&>8Qy@Eweq=Ne8rKsjJTe)mfDw?scqlc&US2dxU0@o5$(Zu(GB4 zujr5^yZdwlP>E{wrkq=NiW~PQZm5`fJz5m&9I}B^zPVNSSa9vWcXu^m%+bU|aOg5q zK%|a72J^vxGy)&3GlNod=Wt|FBG=mgP)o%{(2PCL$9s$dMvIcv^FdM?hbNYQrX%I| z{binoW_?J27M3L2H_Y4n0!3PGL#b*UxRbpd3l$RLC#I})-32((m#4}vP%kHB3Q7PGLpvuro4~7i2u6z$3ar+YSP2?_%+^%f* zR}5Rl@nUnDVdT&uE_ZP%NU-(Zn*^k2*4S;xubW_f3f-cK+=>uy-sK;&F{mRdpgwIgSHfJSw=22paH-mu>R=3Kf9cR*A_Sjg7q#MM< zqobyHu#q_oM3;REOf&nTGa=n6MK4QZ{pey;iGwX&bnAUCVq`=c0{gykLm{VZo%ulF z*n_LEk%}KbmVW1)L+Ab3sSZPR+Fe*5p$^HC|Oyb{_is> zsuD42;l;BT-a#X6fP(~C+`TP&(``5KD7dp9)GD&EVfNN4Bf@5N63j4c_IOZZ`^gF1 zphj9>;b1JVOWrk`HhO{mmk*Lp>wXpL*r|VQth!^2ajO2-Q$=;E0ZcMzj9V;D}3k7ej?g$MEOSvfr*p<&b z6B?7p3F^a78y9pEd$#q2Pm1b zU#?c^Op~TXSZ`3z2a{A=UzcS`zB%Z|XG2xth@1`h=wY$wyp|u2)s&QN#af+k>`vF! z&{oB;K{Wblwtcc`JH%E!TwV2q%vd}p>iZ9d@C(kwR>Dm)p? zV-i0tv8PP66)jD1#I*Qm*`@U`^o)}|58+bGD1y(EEM_dJh-O9xP^xdF-_Z#qZ&m{c zbC6W;iNU!24Cvnj14>>_V8a{IB$GXu&z39rEKNX_07*3xp*W3rJo!}pp2M0Hwe$#* zi#HgV_>>SSD;YT=uK8*Lu|$a+IIXPF$${!eaPU%X#jh@y96VcWEFGqB#<_hE8QPmQ zO_C$p_nXzGgQtqVrC1t-5`*juoj0Q%VLnw`@Yt&eCg!x)84Pq&N%`@t**O@LYz3OR(@+})Hu&$>gJ;6oxdO{ z&KR3!hDx52>YBb*JE@4B`8}j*yOg=37>&zbSN}#T@GA6n9+dFcA*9q_l2eI%Xh*7~ ziU87?k{%5!@e5oasj8xTY|ysPyOMR3W;w?vvG}prD%~$8wf$j!6&K4LI%aD1$6B&8 zG|Bq_{em<75I~pVeMNJ6Dv9e{<=x@Es?2r|L;d(lJhNv+5~$`ps7`1lAq>B{Ot5Ga z6qD6CeNHKADuYBeC(!$C>E5yJ7O5IFfdN*2lPV*LTj(fX$`T*h6!l7_BFQ%HhbJFp zKUVk@Dl`5ZH)LoQ^{7N6?HyY_;Jo?*Uu#dn_XW`49o!xdK!+JJN_3KD7k@2J((0h0 z?0!++a*3VkR_Y8-s+o<1M(>PCz=|sJMqa z0+r0sNH_$gvD_@AC}TCb8}m~2v}_leWOtWdheZwxJl0i{OGIRcO0iVJ-B>5CgP^O-M7OYVJ*8(0|euX~UGp`sq@@gaEw*bHD4*Dj8_ zPO4*=dce-k-f;9Xl`P>A2U6SzIPhFWQT>2(PjqTMlBf}zL3<&dS*!E0mM}&jbXhc- zAb9}5!V(`=H1zl4fM|8TdAE{XwAuTJ>dTw3o}wzSb&xhxCijhe4Q#{|l(FXGy+A)j zH>IZrWy4|#?wJ-1?zBm;cKLHK*H5ngXeiJE?k?6Lz1i+02rcMG7kNDQlDJ_??0D#; z(Bju>vbV@>IGl97vC?TD(|fa!E?NjDA;*m&#_ZiX>Vgi+wr`atYOngkRp_w%?M~sv zUVImV4>dX4Ih+MO4LU`Ui=K%20a~JOwq1$6)KUw@81y#uUGKMV4>O0ioDGDvtZ{Jl zmay)x!zLD>Hl1jqnzX9b_da}w9xr9S`kQwUZPAei4I5Ao#$N}f9I10=!}MXIF!F!C z6+i+ofRKI2Rvlk8erCmgYu2%A6S_nSX7!cGJQ6pQ{xw*Iw(KXQGft90Ft(YQ<7nw! ROz*Khv5A{`^It3We*oUlR=)rM From 535f0707f7ac93a4f4a508566cf4625a7722cab4 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Tue, 2 Jan 2024 14:26:42 -0800 Subject: [PATCH 8/8] Revert "Add ensure green to single node test" This reverts commit 0d6222ca28089295b00c9ef0dedb27e90b784297. Signed-off-by: John Mazanec --- .../TrainingJobRouteDecisionInfoTransportActionTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java index ccf69b423d..9f64afebb4 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java @@ -51,8 +51,6 @@ public void teardown() { @SuppressWarnings("unchecked") public void testNodeOperation() throws IOException, InterruptedException, ExecutionException { - ensureGreen(); - // Ensure initial value of train job count is 0 TrainingJobRouteDecisionInfoTransportAction action = node().injector() .getInstance(TrainingJobRouteDecisionInfoTransportAction.class);