From 3ab1f787758292492d2199ddff13515d1f93b1a1 Mon Sep 17 00:00:00 2001 From: Zelin Hao <87548827+zelinh@users.noreply.github.com> Date: Fri, 6 Jan 2023 16:03:04 -0800 Subject: [PATCH 01/12] Add BWC tests for running against distribution bundle. (#1209) Signed-off-by: Zelin Hao --- integ-test/build.gradle | 109 +++++++++++++----- .../test/resources/security/esnode-key.pem | 28 +++++ .../src/test/resources/security/esnode.pem | 28 +++++ .../src/test/resources/security/kirk-key.pem | 28 +++++ .../src/test/resources/security/kirk.pem | 26 +++++ .../src/test/resources/security/root-ca.pem | 24 ++++ .../src/test/resources/security/sample.pem | 28 +++++ 7 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 integ-test/src/test/resources/security/esnode-key.pem create mode 100644 integ-test/src/test/resources/security/esnode.pem create mode 100644 integ-test/src/test/resources/security/kirk-key.pem create mode 100644 integ-test/src/test/resources/security/kirk.pem create mode 100644 integ-test/src/test/resources/security/root-ca.pem create mode 100644 integ-test/src/test/resources/security/sample.pem diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 900bb1ae90..5fbdbb5d93 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -245,7 +245,12 @@ task compileJdbc(type: Exec) { } } -String bwcVersion = "1.1.0.0"; +String bwcMinVersion = "1.1.0.0" +String bwcBundleVersion = "1.3.2.0" +Boolean bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null && + project.properties['customDistributionDownloadType'] == "bundle"); +String bwcVersion = bwcBundleTest ? bwcBundleVersion : bwcMinVersion +String currentVersion = opensearch_version.replace("-SNAPSHOT","") String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" String bwcSqlPlugin = "opensearch-sql-" + bwcVersion + ".zip" @@ -255,27 +260,55 @@ String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/2021 testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" - versions = ["1.1.0", opensearch_version] numberOfNodes = 3 - plugin(provider(new Callable() { - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - File dir = new File('./integ-test/' + bwcFilePath + bwcVersion) - if (!dir.exists()) { - dir.mkdirs() - } - File f = new File(dir, bwcSqlPlugin) - if (!f.exists()) { - new URL(bwcRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }} + if (bwcBundleTest) { + versions = ["1.3.2", currentVersion] + nodes.each { node -> + node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem")) + node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem")) + node.extraConfigFile("esnode.pem", file("src/test/resources/security/esnode.pem")) + node.extraConfigFile("esnode-key.pem", file("src/test/resources/security/esnode-key.pem")) + node.extraConfigFile("root-ca.pem", file("src/test/resources/security/root-ca.pem")) + node.setting("plugins.security.disabled", "true") + node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") + node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") + node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") + node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") + node.setting("plugins.security.ssl.http.enabled", "true") + node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem") + node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem") + node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") + node.setting("plugins.security.allow_unsafe_democertificates", "true") + node.setting("plugins.security.allow_default_init_securityindex", "true") + node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de") + node.setting("plugins.security.audit.type", "internal_elasticsearch") + node.setting("plugins.security.enable_snapshot_restore_privilege", "true") + node.setting("plugins.security.check_snapshot_restore_write_privileges", "true") + node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]") + node.setting("plugins.security.system_indices.enabled", "true") + } + } else { + versions = ["1.1.0", opensearch_version] + plugin(provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + File dir = new File('./integ-test/' + bwcFilePath + bwcVersion) + if (!dir.exists()) { + dir.mkdirs() + } + File f = new File(dir, bwcSqlPlugin) + if (!f.exists()) { + new URL(bwcRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }} + } + return fileTree(bwcFilePath + bwcVersion).getSingleFile() } - return fileTree(bwcFilePath + bwcVersion).getSingleFile() } } - } - })) + })) + } setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' } @@ -317,8 +350,14 @@ List> plugins = [ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}0" dependsOn "${baseName}#oldVersionClusterTask0" - doFirst { - testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + if (bwcBundleTest){ + doFirst { + testClusters."${baseName}0".nextNodeToNextVersion() + } + } else { + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } } filter { includeTestsMatching "org.opensearch.sql.bwc.*IT" @@ -336,8 +375,14 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { dependsOn "${baseName}#mixedClusterTask" useCluster testClusters."${baseName}0" - doFirst { - testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + if (bwcBundleTest){ + doFirst { + testClusters."${baseName}0".nextNodeToNextVersion() + } + } else { + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } } filter { includeTestsMatching "org.opensearch.sql.bwc.*IT" @@ -355,8 +400,14 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { dependsOn "${baseName}#twoThirdsUpgradedClusterTask" useCluster testClusters."${baseName}0" - doFirst { - testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + if (bwcBundleTest){ + doFirst { + testClusters."${baseName}0".nextNodeToNextVersion() + } + } else { + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } } filter { includeTestsMatching "org.opensearch.sql.bwc.*IT" @@ -374,8 +425,14 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) { dependsOn "${baseName}#oldVersionClusterTask1" useCluster testClusters."${baseName}1" - doFirst { - testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) + if (bwcBundleTest){ + doFirst { + testClusters."${baseName}1".goToNextVersion() + } + } else { + doFirst { + testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) + } } filter { includeTestsMatching "org.opensearch.sql.bwc.*IT" diff --git a/integ-test/src/test/resources/security/esnode-key.pem b/integ-test/src/test/resources/security/esnode-key.pem new file mode 100644 index 0000000000..4ac2cb57a7 --- /dev/null +++ b/integ-test/src/test/resources/security/esnode-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCWvn+O+rywfgMC +ud24mAclMDfuNA/IzCKLxl5usIE/PvUm7PPfXQ14LfQhNQXqOuaD9fiVM+HO1BzK +wmN3j4g7eHInR1cxENoNGKFa0Fr9EXnUv8sfwyobPD8NTu9eaH7T+d6f9oow+Q4n +xb9Xin5IRR/pcJ8v7zEjcXpZaZejcSU4iVZ0PR2Di4H9rfe9SEyR5wLrsVBePB3L +jaL1uK4bZF3n/JGgDe3BNy1PgPU+O+FCzQipBBTyJWQCjd4iTRXVbMa01PglAR85 +O9w6NXApBLyWdGRY6dGd8vMC2P4KlhnxlcgPZdglKniGTX+eTzT7Rszq77zjYrou +PLwSh9S7AgMBAAECggEABwiohxFoEIwws8XcdKqTWsbfNTw0qFfuHLuK2Htf7IWR +htlzn66F3F+4jnwc5IsPCoVFriCXnsEC/usHHSMTZkL+gJqxlNaGdin6DXS/aiOQ +nb69SaQfqNmsz4ApZyxVDqsQGkK0vAhDAtQVU45gyhp/nLLmmqP8lPzMirOEodmp +U9bA8t/ttrzng7SVAER42f6IVpW0iTKTLyFii0WZbq+ObViyqib9hVFrI6NJuQS+ +IelcZB0KsSi6rqIjXg1XXyMiIUcSlhq+GfEa18AYgmsbPwMbExate7/8Ci7ZtCbh +lx9bves2+eeqq5EMm3sMHyhdcg61yzd5UYXeZhwJkQKBgQDS9YqrAtztvLY2gMgv +d+wOjb9awWxYbQTBjx33kf66W+pJ+2j8bI/XX2CpZ98w/oq8VhMqbr9j5b8MfsrF +EoQvedA4joUo8sXd4j1mR2qKF4/KLmkgy6YYusNP2UrVSw7sh77bzce+YaVVoO/e +0wIVTHuD/QZ6fG6MasOqcbl6hwKBgQC27cQruaHFEXR/16LrMVAX+HyEEv44KOCZ +ij5OE4P7F0twb+okngG26+OJV3BtqXf0ULlXJ+YGwXCRf6zUZkld3NMy3bbKPgH6 +H/nf3BxqS2tudj7+DV52jKtisBghdvtlKs56oc9AAuwOs37DvhptBKUPdzDDqfys +Qchv5JQdLQKBgERev+pcqy2Bk6xmYHrB6wdseS/4sByYeIoi0BuEfYH4eB4yFPx6 +UsQCbVl6CKPgWyZe3ydJbU37D8gE78KfFagtWoZ56j4zMF2RDUUwsB7BNCDamce/ +OL2bCeG/Erm98cBG3lxufOX+z47I8fTNfkdY2k8UmhzoZwurLm73HJ3RAoGBAKsp +6yamuXF2FbYRhUXgjHsBbTD/vJO72/yO2CGiLRpi/5mjfkjo99269trp0C8sJSub +5PBiSuADXFsoRgUv+HI1UAEGaCTwxFTQWrRWdtgW3d0sE2EQDVWL5kmfT9TwSeat +mSoyAYR5t3tCBNkPJhbgA7pm4mASzHQ50VyxWs25AoGBAKPFx9X2oKhYQa+mW541 +bbqRuGFMoXIIcr/aeM3LayfLETi48o5NDr2NDP11j4yYuz26YLH0Dj8aKpWuehuH +uB27n6j6qu0SVhQi6mMJBe1JrKbzhqMKQjYOoy8VsC2gdj5pCUP/kLQPW7zm9diX +CiKTtKgPIeYdigor7V3AHcVT +-----END PRIVATE KEY----- diff --git a/integ-test/src/test/resources/security/esnode.pem b/integ-test/src/test/resources/security/esnode.pem new file mode 100644 index 0000000000..7ba92534e4 --- /dev/null +++ b/integ-test/src/test/resources/security/esnode.pem @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIEyTCCA7GgAwIBAgIGAWLrc1O2MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm +iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ +RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 +IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwHhcNMTgwNDIy +MDM0MzQ3WhcNMjgwNDE5MDM0MzQ3WjBeMRIwEAYKCZImiZPyLGQBGRYCZGUxDTAL +BgNVBAcMBHRlc3QxDTALBgNVBAoMBG5vZGUxDTALBgNVBAsMBG5vZGUxGzAZBgNV +BAMMEm5vZGUtMC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAJa+f476vLB+AwK53biYByUwN+40D8jMIovGXm6wgT8+9Sbs899dDXgt +9CE1Beo65oP1+JUz4c7UHMrCY3ePiDt4cidHVzEQ2g0YoVrQWv0RedS/yx/DKhs8 +Pw1O715oftP53p/2ijD5DifFv1eKfkhFH+lwny/vMSNxellpl6NxJTiJVnQ9HYOL +gf2t971ITJHnAuuxUF48HcuNovW4rhtkXef8kaAN7cE3LU+A9T474ULNCKkEFPIl +ZAKN3iJNFdVsxrTU+CUBHzk73Do1cCkEvJZ0ZFjp0Z3y8wLY/gqWGfGVyA9l2CUq +eIZNf55PNPtGzOrvvONiui48vBKH1LsCAwEAAaOCAVkwggFVMIG8BgNVHSMEgbQw +gbGAFJI1DOAPHitF9k0583tfouYSl0BzoYGVpIGSMIGPMRMwEQYKCZImiZPyLGQB +GRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhhbXBs +ZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMSEw +HwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCAQEwHQYDVR0OBBYEFKyv +78ZmFjVKM9g7pMConYH7FVBHMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgXg +MCAGA1UdJQEB/wQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA1BgNVHREELjAsiAUq +AwQFBYISbm9kZS0wLmV4YW1wbGUuY29tgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZI +hvcNAQELBQADggEBAIOKuyXsFfGv1hI/Lkpd/73QNqjqJdxQclX57GOMWNbOM5H0 +5/9AOIZ5JQsWULNKN77aHjLRr4owq2jGbpc/Z6kAd+eiatkcpnbtbGrhKpOtoEZy +8KuslwkeixpzLDNISSbkeLpXz4xJI1ETMN/VG8ZZP1bjzlHziHHDu0JNZ6TnNzKr +XzCGMCohFfem8vnKNnKUneMQMvXd3rzUaAgvtf7Hc2LTBlf4fZzZF1EkwdSXhaMA +1lkfHiqOBxtgeDLxCHESZ2fqgVqsWX+t3qHQfivcPW6txtDyrFPRdJOGhiMGzT/t +e/9kkAtQRgpTb3skYdIOOUOV0WGQ60kJlFhAzIs= +-----END CERTIFICATE----- diff --git a/integ-test/src/test/resources/security/kirk-key.pem b/integ-test/src/test/resources/security/kirk-key.pem new file mode 100644 index 0000000000..bacb22c215 --- /dev/null +++ b/integ-test/src/test/resources/security/kirk-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDCwgBOoO88uMM8 +dREJsk58Yt4Jn0zwQ2wUThbvy3ICDiEWhiAhUbg6dTggpS5vWWJto9bvaaqgMVoh +ElfYHdTDncX3UQNBEP8tqzHON6BFEFSGgJRGLd6f5dri6rK32nCotYS61CFXBFxf +WumXjSukjyrcTsdkR3C5QDo2oN7F883MOQqRENPzAtZi9s3jNX48u+/e3yvJzXsB +GS9Qmsye6C71enbIujM4CVwDT/7a5jHuaUp6OuNCFbdRPnu/wLYwOS2/yOtzAqk7 +/PFnPCe7YOa10ShnV/jx2sAHhp7ZQBJgFkkgnIERz9Ws74Au+EbptWnsWuB+LqRL +x5G02IzpAgMBAAECggEAEzwnMkeBbqqDgyRqFbO/PgMNvD7i0b/28V0dCtCPEVY6 +klzrg3RCERP5V9AN8VVkppYjPkCzZ2A4b0JpMUu7ncOmr7HCnoSCj2IfEyePSVg+ +4OHbbcBOAoDTHiI2myM/M9++8izNS34qGV4t6pfjaDyeQQ/5cBVWNBWnKjS34S5H +rJWpAcDgxYk5/ah2Xs2aULZlXDMxbSikjrv+n4JIYTKFQo8ydzL8HQDBRmXAFLjC +gNOSHf+5u1JdpY3uPIxK1ugVf8zPZ4/OEB23j56uu7c8+sZ+kZwfRWAQmMhFVG/y +OXxoT5mOruBsAw29m2Ijtxg252/YzSTxiDqFziB/eQKBgQDjeVAdi55GW/bvhuqn +xME/An8E3hI/FyaaITrMQJUBjiCUaStTEqUgQ6A7ZfY/VX6qafOX7sli1svihrXC +uelmKrdve/CFEEqzX9JWWRiPiQ0VZD+EQRsJvX85Tw2UGvVUh6dO3UGPS0BhplMD +jeVpyXgZ7Gy5we+DWjfwhYrCmwKBgQDbLmQhRy+IdVljObZmv3QtJ0cyxxZETWzU +MKmgBFvcRw+KvNwO+Iy0CHEbDu06Uj63kzI2bK3QdINaSrjgr8iftXIQpBmcgMF+ +a1l5HtHlCp6RWd55nWQOEvn36IGN3cAaQkXuh4UYM7QfEJaAbzJhyJ+wXA3jWqUd +8bDTIAZ0ywKBgFuZ44gyTAc7S2JDa0Up90O/ZpT4NFLRqMrSbNIJg7d/m2EIRNkM +HhCzCthAg/wXGo3XYq+hCdnSc4ICCzmiEfoBY6LyPvXmjJ5VDOeWs0xBvVIK74T7 +jr7KX2wdiHNGs9pZUidw89CXVhK8nptEzcheyA1wZowbK68yamph7HHXAoGBAK3x +7D9Iyl1mnDEWPT7f1Gh9UpDm1TIRrDvd/tBihTCVKK13YsFy2d+LD5Bk0TpGyUVR +STlOGMdloFUJFh4jA3pUOpkgUr8Uo/sbYN+x6Ov3+I3sH5aupRhSURVA7YhUIz/z +tqIt5R+m8Nzygi6dkQNvf+Qruk3jw0S3ahizwsvvAoGAL7do6dTLp832wFVxkEf4 +gg1M6DswfkgML5V/7GQ3MkIX/Hrmiu+qSuHhDGrp9inZdCDDYg5+uy1+2+RBMRZ3 +vDUUacvc4Fep05zp7NcjgU5y+/HWpuKVvLIlZAO1MBY4Xinqqii6RdxukIhxw7eT +C6TPL5KAcV1R/XAihDhI18Y= +-----END PRIVATE KEY----- diff --git a/integ-test/src/test/resources/security/kirk.pem b/integ-test/src/test/resources/security/kirk.pem new file mode 100644 index 0000000000..c32b21cd89 --- /dev/null +++ b/integ-test/src/test/resources/security/kirk.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEdzCCA1+gAwIBAgIGAWLrc1O4MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm +iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ +RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 +IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwHhcNMTgwNDIy +MDM0MzQ3WhcNMjgwNDE5MDM0MzQ3WjBNMQswCQYDVQQGEwJkZTENMAsGA1UEBwwE +dGVzdDEPMA0GA1UECgwGY2xpZW50MQ8wDQYDVQQLDAZjbGllbnQxDTALBgNVBAMM +BGtpcmswggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDCwgBOoO88uMM8 +dREJsk58Yt4Jn0zwQ2wUThbvy3ICDiEWhiAhUbg6dTggpS5vWWJto9bvaaqgMVoh +ElfYHdTDncX3UQNBEP8tqzHON6BFEFSGgJRGLd6f5dri6rK32nCotYS61CFXBFxf +WumXjSukjyrcTsdkR3C5QDo2oN7F883MOQqRENPzAtZi9s3jNX48u+/e3yvJzXsB +GS9Qmsye6C71enbIujM4CVwDT/7a5jHuaUp6OuNCFbdRPnu/wLYwOS2/yOtzAqk7 +/PFnPCe7YOa10ShnV/jx2sAHhp7ZQBJgFkkgnIERz9Ws74Au+EbptWnsWuB+LqRL +x5G02IzpAgMBAAGjggEYMIIBFDCBvAYDVR0jBIG0MIGxgBSSNQzgDx4rRfZNOfN7 +X6LmEpdAc6GBlaSBkjCBjzETMBEGCgmSJomT8ixkARkWA2NvbTEXMBUGCgmSJomT +8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1wbGUgQ29tIEluYy4xITAfBgNV +BAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEhMB8GA1UEAwwYRXhhbXBsZSBD +b20gSW5jLiBSb290IENBggEBMB0GA1UdDgQWBBRsdhuHn3MGDvZxOe22+1wliCJB +mDAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggr +BgEFBQcDAjANBgkqhkiG9w0BAQsFAAOCAQEAkPrUTKKn+/6g0CjhTPBFeX8mKXhG +zw5z9Oq+xnwefZwxV82E/tgFsPcwXcJIBg0f43BaVSygPiV7bXqWhxASwn73i24z +lveIR4+z56bKIhP6c3twb8WWR9yDcLu2Iroin7dYEm3dfVUrhz/A90WHr6ddwmLL +3gcFF2kBu3S3xqM5OmN/tqRXFmo+EvwrdJRiTh4Fsf0tX1ZT07rrGvBFYktK7Kma +lqDl4UDCF1UWkiiFubc0Xw+DR6vNAa99E0oaphzvCmITU1wITNnYZTKzVzQ7vUCq +kLmXOFLTcxTQpptxSo5xDD3aTpzWGCvjExCKpXQtsITUOYtZc02AGjjPOQ== +-----END CERTIFICATE----- diff --git a/integ-test/src/test/resources/security/root-ca.pem b/integ-test/src/test/resources/security/root-ca.pem new file mode 100644 index 0000000000..4015d866e1 --- /dev/null +++ b/integ-test/src/test/resources/security/root-ca.pem @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/jCCAuagAwIBAgIBATANBgkqhkiG9w0BAQsFADCBjzETMBEGCgmSJomT8ixk +ARkWA2NvbTEXMBUGCgmSJomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1w +bGUgQ29tIEluYy4xITAfBgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEh +MB8GA1UEAwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMB4XDTE4MDQyMjAzNDM0 +NloXDTI4MDQxOTAzNDM0NlowgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJ +kiaJk/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEw +HwYDVQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1w +bGUgQ29tIEluYy4gUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAK/u+GARP5innhpXK0c0q7s1Su1VTEaIgmZr8VWI6S8amf5cU3ktV7WT9SuV +TsAm2i2A5P+Ctw7iZkfnHWlsC3HhPUcd6mvzGZ4moxnamM7r+a9otRp3owYoGStX +ylVTQusAjbq9do8CMV4hcBTepCd+0w0v4h6UlXU8xjhj1xeUIz4DKbRgf36q0rv4 +VIX46X72rMJSETKOSxuwLkov1ZOVbfSlPaygXIxqsHVlj1iMkYRbQmaTib6XWHKf +MibDaqDejOhukkCjzpptGZOPFQ8002UtTTNv1TiaKxkjMQJNwz6jfZ53ws3fh1I0 +RWT6WfM4oeFRFnyFRmc4uYTUgAkCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAf +BgNVHSMEGDAWgBSSNQzgDx4rRfZNOfN7X6LmEpdAczAdBgNVHQ4EFgQUkjUM4A8e +K0X2TTnze1+i5hKXQHMwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4IB +AQBoQHvwsR34hGO2m8qVR9nQ5Klo5HYPyd6ySKNcT36OZ4AQfaCGsk+SecTi35QF +RHL3g2qffED4tKR0RBNGQSgiLavmHGCh3YpDupKq2xhhEeS9oBmQzxanFwWFod4T +nnsG2cCejyR9WXoRzHisw0KJWeuNlwjUdJY0xnn16srm1zL/M/f0PvCyh9HU1mF1 +ivnOSqbDD2Z7JSGyckgKad1Omsg/rr5XYtCeyJeXUPcmpeX6erWJJNTUh6yWC/hY +G/dFC4xrJhfXwz6Z0ytUygJO32bJG4Np2iGAwvvgI9EfxzEv/KP+FGrJOvQJAq4/ +BU36ZAa80W/8TBnqZTkNnqZV +-----END CERTIFICATE----- diff --git a/integ-test/src/test/resources/security/sample.pem b/integ-test/src/test/resources/security/sample.pem new file mode 100644 index 0000000000..7ba92534e4 --- /dev/null +++ b/integ-test/src/test/resources/security/sample.pem @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIEyTCCA7GgAwIBAgIGAWLrc1O2MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm +iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ +RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 +IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwHhcNMTgwNDIy +MDM0MzQ3WhcNMjgwNDE5MDM0MzQ3WjBeMRIwEAYKCZImiZPyLGQBGRYCZGUxDTAL +BgNVBAcMBHRlc3QxDTALBgNVBAoMBG5vZGUxDTALBgNVBAsMBG5vZGUxGzAZBgNV +BAMMEm5vZGUtMC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAJa+f476vLB+AwK53biYByUwN+40D8jMIovGXm6wgT8+9Sbs899dDXgt +9CE1Beo65oP1+JUz4c7UHMrCY3ePiDt4cidHVzEQ2g0YoVrQWv0RedS/yx/DKhs8 +Pw1O715oftP53p/2ijD5DifFv1eKfkhFH+lwny/vMSNxellpl6NxJTiJVnQ9HYOL +gf2t971ITJHnAuuxUF48HcuNovW4rhtkXef8kaAN7cE3LU+A9T474ULNCKkEFPIl +ZAKN3iJNFdVsxrTU+CUBHzk73Do1cCkEvJZ0ZFjp0Z3y8wLY/gqWGfGVyA9l2CUq +eIZNf55PNPtGzOrvvONiui48vBKH1LsCAwEAAaOCAVkwggFVMIG8BgNVHSMEgbQw +gbGAFJI1DOAPHitF9k0583tfouYSl0BzoYGVpIGSMIGPMRMwEQYKCZImiZPyLGQB +GRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhhbXBs +ZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMSEw +HwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCAQEwHQYDVR0OBBYEFKyv +78ZmFjVKM9g7pMConYH7FVBHMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgXg +MCAGA1UdJQEB/wQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA1BgNVHREELjAsiAUq +AwQFBYISbm9kZS0wLmV4YW1wbGUuY29tgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZI +hvcNAQELBQADggEBAIOKuyXsFfGv1hI/Lkpd/73QNqjqJdxQclX57GOMWNbOM5H0 +5/9AOIZ5JQsWULNKN77aHjLRr4owq2jGbpc/Z6kAd+eiatkcpnbtbGrhKpOtoEZy +8KuslwkeixpzLDNISSbkeLpXz4xJI1ETMN/VG8ZZP1bjzlHziHHDu0JNZ6TnNzKr +XzCGMCohFfem8vnKNnKUneMQMvXd3rzUaAgvtf7Hc2LTBlf4fZzZF1EkwdSXhaMA +1lkfHiqOBxtgeDLxCHESZ2fqgVqsWX+t3qHQfivcPW6txtDyrFPRdJOGhiMGzT/t +e/9kkAtQRgpTb3skYdIOOUOV0WGQ60kJlFhAzIs= +-----END CERTIFICATE----- From fdb45cf2382211f29c57598e80dc918876f6a378 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Fri, 6 Jan 2023 10:36:52 -0800 Subject: [PATCH 02/12] Add Alternate Syntax For Match_Query And Other Functions (#1166) Added Tests And Implementation For Match_Query, Match_Phrase, and Multi_Match Functions Signed-off-by: GabeFernandez310 Signed-off-by: GabeFernandez310 Signed-off-by: GabeFernandez310 --- docs/user/dql/functions.rst | 55 +++++++++++++ .../opensearch/sql/legacy/MethodQueryIT.java | 2 +- .../java/org/opensearch/sql/sql/MatchIT.java | 33 +++++++- .../org/opensearch/sql/sql/MatchPhraseIT.java | 28 ++++++- .../org/opensearch/sql/sql/MultiMatchIT.java | 31 +++++++ sql/src/main/antlr/OpenSearchSQLParser.g4 | 27 ++++++- .../sql/sql/parser/AstExpressionBuilder.java | 43 ++++++++++ .../sql/sql/antlr/SQLSyntaxParserTest.java | 24 ++++++ .../sql/parser/AstExpressionBuilderTest.java | 81 +++++++++++++++++++ 9 files changed, 320 insertions(+), 4 deletions(-) diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 7f7284c95b..1726c92054 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -3044,6 +3044,16 @@ Another example to show how to set custom values for the optional parameters:: | Bond | +------------+ + The matchquery function also supports an alternative syntax:: + + os> SELECT firstname FROM accounts WHERE firstname = matchquery('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + MATCH_QUERY ----- @@ -3073,6 +3083,16 @@ Another example to show how to set custom values for the optional parameters:: | Bond | +------------+ +The match_query function also supports an alternative syntax:: + + os> SELECT firstname FROM accounts WHERE firstname = match_query('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + MATCH_PHRASE ------------ @@ -3112,6 +3132,23 @@ Another example to show how to set custom values for the optional parameters:: | Alan Alexander Milne | Winnie-the-Pooh | +----------------------+--------------------------+ +The match_phrase function also supports an alternative syntax:: + + os> SELECT firstname FROM accounts WHERE firstname = match_phrase('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + + os> SELECT firstname FROM accounts WHERE firstname = matchphrase('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ MATCH_BOOL_PREFIX ----- @@ -3255,6 +3292,24 @@ Another example to show how to set custom values for the optional parameters:: | 1 | The House at Pooh Corner | Alan Alexander Milne | +------+--------------------------+----------------------+ +The multi_match function also supports an alternative syntax:: + + os> SELECT firstname FROM accounts WHERE firstname = multi_match('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + + os> SELECT firstname FROM accounts WHERE firstname = multimatch('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + SIMPLE_QUERY_STRING ------------------- diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java index b1b6b24ada..fdbbb0f6ba 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java @@ -54,7 +54,7 @@ public void matchQueryTest() throws IOException { "select address from %s where address= matchQuery('880 Holmes Lane') limit 3", TestsConstants.TEST_INDEX_ACCOUNT)); Assert.assertThat(result, - containsString("{\"match\":{\"address\":{\"query\":\"880 Holmes Lane\"")); + containsString("{\\\"match\\\":{\\\"address\\\":{\\\"query\\\":\\\"880 Holmes Lane\\\"")); } /** diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java index b113e83477..28573fdd10 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java @@ -103,7 +103,7 @@ public void match_query_in_having() throws IOException { } @Test - public void alternate_syntaxes_return_the_same_results() throws IOException { + public void match_aliases_return_the_same_results() throws IOException { String query1 = "SELECT lastname FROM " + TEST_INDEX_ACCOUNT + " HAVING match(firstname, 'Nanette')"; JSONObject result1 = executeJdbcRequest(query1); @@ -116,4 +116,35 @@ public void alternate_syntaxes_return_the_same_results() throws IOException { assertEquals(result1.getInt("total"), result2.getInt("total")); assertEquals(result1.getInt("total"), result3.getInt("total")); } + + @Test + public void match_query_alternate_syntax() throws IOException { + JSONObject result = executeJdbcRequest( + "SELECT lastname FROM " + TEST_INDEX_ACCOUNT + " WHERE lastname = match_query('Bates')"); + verifySchema(result, schema("lastname", "text")); + verifyDataRows(result, rows("Bates")); + } + + @Test + public void matchquery_alternate_syntax() throws IOException { + JSONObject result = executeJdbcRequest( + "SELECT lastname FROM " + TEST_INDEX_ACCOUNT + " WHERE lastname = matchquery('Bates')"); + verifySchema(result, schema("lastname", "text")); + verifyDataRows(result, rows("Bates")); + } + + @Test + public void match_alternate_syntaxes_return_the_same_results() throws IOException { + String query1 = "SELECT * FROM " + + TEST_INDEX_ACCOUNT + " WHERE match(firstname, 'Nanette')"; + JSONObject result1 = executeJdbcRequest(query1); + String query2 = "SELECT * FROM " + + TEST_INDEX_ACCOUNT + " WHERE firstname = match_query('Nanette')"; + JSONObject result2 = executeJdbcRequest(query2); + String query3 = "SELECT * FROM " + + TEST_INDEX_ACCOUNT + " WHERE firstname = matchquery('Nanette')"; + JSONObject result3 = executeJdbcRequest(query3); + assertEquals(result1.getInt("total"), result2.getInt("total")); + assertEquals(result1.getInt("total"), result3.getInt("total")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java index b870a60604..3b7e65dcc6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java @@ -51,7 +51,7 @@ public void test_match_phrase_with_slop() throws IOException { } @Test - public void test_alternate_syntax_for_match_phrase_returns_same_result() throws IOException { + public void test_aliases_for_match_phrase_returns_same_result() throws IOException { String query1 = "SELECT phrase FROM %s WHERE matchphrase(phrase, 'quick fox')"; String query2 = "SELECT phrase FROM %s WHERE match_phrase(phrase, 'quick fox')"; String query3 = "SELECT phrase FROM %s WHERE matchphrasequery(phrase, 'quick fox')"; @@ -61,4 +61,30 @@ public void test_alternate_syntax_for_match_phrase_returns_same_result() throws assertTrue(result1.similar(result2)); assertTrue(result1.similar(result3)); } + + @Test + public void match_phrase_alternate_syntax() throws IOException { + String query = "SELECT phrase FROM %s WHERE phrase = match_phrase('quick fox')"; + JSONObject result = executeJdbcRequest(String.format(query, TEST_INDEX_PHRASE)); + verifyDataRows(result, rows("quick fox"), rows("quick fox here")); + } + + @Test + public void matchphrase_alternate_syntax() throws IOException { + String query = "SELECT phrase FROM %s WHERE phrase = matchphrase('quick fox')"; + JSONObject result = executeJdbcRequest(String.format(query, TEST_INDEX_PHRASE)); + verifyDataRows(result, rows("quick fox"), rows("quick fox here")); + } + + @Test + public void match_phrase_alternate_syntaxes_return_the_same_results() throws IOException { + String query1 = "SELECT phrase FROM %s WHERE matchphrase(phrase, 'quick fox')"; + String query2 = "SELECT phrase FROM %s WHERE phrase = matchphrase('quick fox')"; + String query3 = "SELECT phrase FROM %s WHERE phrase = match_phrase('quick fox')"; + JSONObject result1 = executeJdbcRequest(String.format(query1, TEST_INDEX_PHRASE)); + JSONObject result2 = executeJdbcRequest(String.format(query2, TEST_INDEX_PHRASE)); + JSONObject result3 = executeJdbcRequest(String.format(query3, TEST_INDEX_PHRASE)); + assertTrue(result1.similar(result2)); + assertTrue(result1.similar(result3)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java index 24ce45fd20..07c89b4cdf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java @@ -124,4 +124,35 @@ public void test_all_params_multimatchquery_alternate_parameter_syntax() { JSONObject result = executeJdbcRequest(query); assertEquals(2, result.getInt("total")); } + + @Test + public void multi_match_alternate_syntax() throws IOException { + String query = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE CreationDate = multi_match('2014-01-22');"; + var result = new JSONObject(executeQuery(query, "jdbc")); + assertEquals(8, result.getInt("total")); + } + + @Test + public void multimatch_alternate_syntax() throws IOException { + String query = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE CreationDate = multimatch('2014-01-22');"; + var result = new JSONObject(executeQuery(query, "jdbc")); + assertEquals(8, result.getInt("total")); + } + + @Test + public void multi_match_alternate_syntaxes_return_the_same_results() throws IOException { + String query1 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE multi_match(['CreationDate'], '2014-01-22');"; + String query2 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE CreationDate = multi_match('2014-01-22');"; + String query3 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE CreationDate = multimatch('2014-01-22');"; + var result1 = new JSONObject(executeQuery(query1, "jdbc")); + var result2 = new JSONObject(executeQuery(query2, "jdbc")); + var result3 = new JSONObject(executeQuery(query3, "jdbc")); + assertEquals(result1.getInt("total"), result2.getInt("total")); + assertEquals(result1.getInt("total"), result3.getInt("total")); + } } diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 954fc44c3f..f10b6af186 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -328,6 +328,10 @@ positionFunction : POSITION LR_BRACKET functionArg IN functionArg RR_BRACKET ; +matchQueryAltSyntaxFunction + : field=relevanceField EQUAL_SYMBOL MATCH_QUERY LR_BRACKET query=relevanceQuery RR_BRACKET + ; + scalarFunctionName : mathematicalFunctionName | dateTimeFunctionName @@ -345,7 +349,8 @@ specificFunction ; relevanceFunction - : noFieldRelevanceFunction | singleFieldRelevanceFunction | multiFieldRelevanceFunction + : noFieldRelevanceFunction | singleFieldRelevanceFunction | multiFieldRelevanceFunction | altSingleFieldRelevanceFunction | altMultiFieldRelevanceFunction + ; noFieldRelevanceFunction @@ -367,6 +372,14 @@ multiFieldRelevanceFunction alternateMultiMatchQuery COMMA alternateMultiMatchField (COMMA relevanceArg)* RR_BRACKET ; +altSingleFieldRelevanceFunction + : field=relevanceField EQUAL_SYMBOL altSyntaxFunctionName=altSingleFieldRelevanceFunctionName LR_BRACKET query=relevanceQuery (COMMA relevanceArg)* RR_BRACKET + ; + +altMultiFieldRelevanceFunction + : field=relevanceField EQUAL_SYMBOL altSyntaxFunctionName=altMultiFieldRelevanceFunctionName LR_BRACKET query=relevanceQuery (COMMA relevanceArg)* RR_BRACKET + ; + convertedDataType : typeName=DATE | typeName=TIME @@ -487,6 +500,18 @@ multiFieldRelevanceFunctionName | QUERY_STRING ; +altSingleFieldRelevanceFunctionName + : MATCH_QUERY + | MATCHQUERY + | MATCH_PHRASE + | MATCHPHRASE + ; + +altMultiFieldRelevanceFunctionName + : MULTI_MATCH + | MULTIMATCH + ; + functionArgs : (functionArg (COMMA functionArg)*)? ; diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 20ed1322e0..086d1c9ff2 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -433,6 +433,14 @@ public UnresolvedExpression visitSingleFieldRelevanceFunction( singleFieldRelevanceArguments(ctx)); } + @Override + public UnresolvedExpression visitAltSingleFieldRelevanceFunction( + OpenSearchSQLParser.AltSingleFieldRelevanceFunctionContext ctx) { + return new Function( + ctx.altSyntaxFunctionName.getText().toLowerCase(), + altSingleFieldRelevanceFunctionArguments(ctx)); + } + @Override public UnresolvedExpression visitMultiFieldRelevanceFunction( MultiFieldRelevanceFunctionContext ctx) { @@ -454,6 +462,14 @@ public UnresolvedExpression visitMultiFieldRelevanceFunction( } } + @Override + public UnresolvedExpression visitAltMultiFieldRelevanceFunction( + OpenSearchSQLParser.AltMultiFieldRelevanceFunctionContext ctx) { + return new Function( + ctx.altSyntaxFunctionName.getText().toLowerCase(), + altMultiFieldRelevanceFunctionArguments(ctx)); + } + private Function buildFunction(String functionName, List arg) { return new Function( @@ -510,6 +526,18 @@ private List singleFieldRelevanceArguments( } + private List altSingleFieldRelevanceFunctionArguments( + OpenSearchSQLParser.AltSingleFieldRelevanceFunctionContext ctx) { + // all the arguments are defaulted to string values + // to skip environment resolving and function signature resolving + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(new UnresolvedArgument("field", + new Literal(StringUtils.unquoteText(ctx.field.getText()), DataType.STRING))); + builder.add(new UnresolvedArgument("query", + new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); + fillRelevanceArgs(ctx.relevanceArg(), builder); + return builder.build(); + } private List multiFieldRelevanceArguments( MultiFieldRelevanceFunctionContext ctx) { @@ -565,4 +593,19 @@ private List alternateMultiMatchArguments( return builder.build(); } + + private List altMultiFieldRelevanceFunctionArguments( + OpenSearchSQLParser.AltMultiFieldRelevanceFunctionContext ctx) { + // all the arguments are defaulted to string values + // to skip environment resolving and function signature resolving + var map = new HashMap(); + map.put(ctx.field.getText(), 1F); + ImmutableList.Builder builder = ImmutableList.builder(); + var fields = new RelevanceFieldList(map); + builder.add(new UnresolvedArgument("fields", fields)); + builder.add(new UnresolvedArgument("query", + new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); + fillRelevanceArgs(ctx.relevanceArg(), builder); + return builder.build(); + } } diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index 5c9ccaa3ec..d1a8f29a2c 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -509,6 +509,30 @@ private static Stream matchPhraseComplexQueries() { ); } + @Test + public void canParseMatchQueryAlternateSyntax() { + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = matchquery('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = matchquery(\"query\")")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = match_query('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = match_query(\"query\")")); + } + + @Test + public void canParseMatchPhraseAlternateSyntax() { + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = match_phrase('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = match_phrase(\"query\")")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = matchphrase('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = matchphrase(\"query\")")); + } + + @Test + public void canParseMultiMatchAlternateSyntax() { + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = multi_match('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = multi_match(\"query\")")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = multimatch('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = multimatch(\"query\")")); + } + private static Stream matchPhraseQueryComplexQueries() { return Stream.of( "SELECT * FROM t WHERE matchphrasequery(c, 3)", diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index 113a828f0e..0bc44cdffd 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -572,6 +572,87 @@ public void relevanceMatch_Query() { buildExprAst("match_query('message', 'search query', analyzer='keyword', operator='AND')")); } + @Test + public void relevanceMatchQueryAltSyntax() { + assertEquals(AstDSL.function("match_query", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = match_query('search query')") + ); + + assertEquals(AstDSL.function("match_query", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = match_query(\"search query\")") + ); + + assertEquals(AstDSL.function("matchquery", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = matchquery('search query')") + ); + + assertEquals(AstDSL.function("matchquery", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = matchquery(\"search query\")") + ); + } + + @Test + public void relevanceMatchPhraseAltSyntax() { + assertEquals(AstDSL.function("match_phrase", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = match_phrase('search query')") + ); + + assertEquals(AstDSL.function("match_phrase", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = match_phrase(\"search query\")") + ); + + assertEquals(AstDSL.function("matchphrase", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = matchphrase('search query')") + ); + + assertEquals(AstDSL.function("matchphrase", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = matchphrase(\"search query\")") + ); + } + + @Test + public void relevanceMultiMatchAltSyntax() { + assertEquals(AstDSL.function("multi_match", + unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of("field1", 1.F))), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("field1 = multi_match('search query')") + ); + + assertEquals(AstDSL.function("multi_match", + unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of("field1", 1.F))), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("field1 = multi_match(\"search query\")") + ); + + assertEquals(AstDSL.function("multimatch", + unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of("field1", 1.F))), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("field1 = multimatch('search query')") + ); + + assertEquals(AstDSL.function("multimatch", + unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of("field1", 1.F))), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("field1 = multimatch(\"search query\")") + ); + } + @Test public void relevanceMulti_match() { assertEquals(AstDSL.function("multi_match", From 8a676673e4e4e75986af36833b164a8b72fd0aee Mon Sep 17 00:00:00 2001 From: YANGDB Date: Fri, 6 Jan 2023 16:09:04 -0800 Subject: [PATCH 03/12] Merge pull request #1241 from Bit-Quill/Failing-CI-Hot-Fix Hot Fix For CI Build (cherry picked from commit aae57a0bdc995bd53bd88eb812d329761c4333a6) Signed-off-by: GabeFernandez310 --- .../sql/sql/parser/AstExpressionBuilder.java | 14 ++++---- .../sql/parser/AstExpressionBuilderTest.java | 32 +++++++++---------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 086d1c9ff2..48d436cf23 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -16,8 +16,9 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT_LIKE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.POSITION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEXP; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AltMultiFieldRelevanceFunctionContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AltSingleFieldRelevanceFunctionContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AlternateMultiMatchFieldContext; -import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AlternateMultiMatchQueryContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BetweenPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BinaryComparisonPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BooleanContext; @@ -96,6 +97,7 @@ import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AlternateMultiMatchQueryContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AndExpressionContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ColumnNameContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IdentContext; @@ -435,7 +437,7 @@ public UnresolvedExpression visitSingleFieldRelevanceFunction( @Override public UnresolvedExpression visitAltSingleFieldRelevanceFunction( - OpenSearchSQLParser.AltSingleFieldRelevanceFunctionContext ctx) { + AltSingleFieldRelevanceFunctionContext ctx) { return new Function( ctx.altSyntaxFunctionName.getText().toLowerCase(), altSingleFieldRelevanceFunctionArguments(ctx)); @@ -464,7 +466,7 @@ public UnresolvedExpression visitMultiFieldRelevanceFunction( @Override public UnresolvedExpression visitAltMultiFieldRelevanceFunction( - OpenSearchSQLParser.AltMultiFieldRelevanceFunctionContext ctx) { + AltMultiFieldRelevanceFunctionContext ctx) { return new Function( ctx.altSyntaxFunctionName.getText().toLowerCase(), altMultiFieldRelevanceFunctionArguments(ctx)); @@ -527,12 +529,12 @@ private List singleFieldRelevanceArguments( private List altSingleFieldRelevanceFunctionArguments( - OpenSearchSQLParser.AltSingleFieldRelevanceFunctionContext ctx) { + AltSingleFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values // to skip environment resolving and function signature resolving ImmutableList.Builder builder = ImmutableList.builder(); builder.add(new UnresolvedArgument("field", - new Literal(StringUtils.unquoteText(ctx.field.getText()), DataType.STRING))); + new QualifiedName(StringUtils.unquoteText(ctx.field.getText())))); builder.add(new UnresolvedArgument("query", new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); fillRelevanceArgs(ctx.relevanceArg(), builder); @@ -595,7 +597,7 @@ private List alternateMultiMatchArguments( } private List altMultiFieldRelevanceFunctionArguments( - OpenSearchSQLParser.AltMultiFieldRelevanceFunctionContext ctx) { + AltMultiFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values // to skip environment resolving and function signature resolving var map = new HashMap(); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index 0bc44cdffd..23d3ddbc49 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -576,26 +576,26 @@ public void relevanceMatch_Query() { public void relevanceMatchQueryAltSyntax() { assertEquals(AstDSL.function("match_query", unresolvedArg("field", stringLiteral("message")), - unresolvedArg("query", stringLiteral("search query"))), - buildExprAst("message = match_query('search query')") + unresolvedArg("query", stringLiteral("search query"))).toString(), + buildExprAst("message = match_query('search query')").toString() ); assertEquals(AstDSL.function("match_query", unresolvedArg("field", stringLiteral("message")), - unresolvedArg("query", stringLiteral("search query"))), - buildExprAst("message = match_query(\"search query\")") + unresolvedArg("query", stringLiteral("search query"))).toString(), + buildExprAst("message = match_query(\"search query\")").toString() ); assertEquals(AstDSL.function("matchquery", unresolvedArg("field", stringLiteral("message")), - unresolvedArg("query", stringLiteral("search query"))), - buildExprAst("message = matchquery('search query')") + unresolvedArg("query", stringLiteral("search query"))).toString(), + buildExprAst("message = matchquery('search query')").toString() ); assertEquals(AstDSL.function("matchquery", unresolvedArg("field", stringLiteral("message")), - unresolvedArg("query", stringLiteral("search query"))), - buildExprAst("message = matchquery(\"search query\")") + unresolvedArg("query", stringLiteral("search query"))).toString(), + buildExprAst("message = matchquery(\"search query\")").toString() ); } @@ -603,26 +603,26 @@ public void relevanceMatchQueryAltSyntax() { public void relevanceMatchPhraseAltSyntax() { assertEquals(AstDSL.function("match_phrase", unresolvedArg("field", stringLiteral("message")), - unresolvedArg("query", stringLiteral("search query"))), - buildExprAst("message = match_phrase('search query')") + unresolvedArg("query", stringLiteral("search query"))).toString(), + buildExprAst("message = match_phrase('search query')").toString() ); assertEquals(AstDSL.function("match_phrase", unresolvedArg("field", stringLiteral("message")), - unresolvedArg("query", stringLiteral("search query"))), - buildExprAst("message = match_phrase(\"search query\")") + unresolvedArg("query", stringLiteral("search query"))).toString(), + buildExprAst("message = match_phrase(\"search query\")").toString() ); assertEquals(AstDSL.function("matchphrase", unresolvedArg("field", stringLiteral("message")), - unresolvedArg("query", stringLiteral("search query"))), - buildExprAst("message = matchphrase('search query')") + unresolvedArg("query", stringLiteral("search query"))).toString(), + buildExprAst("message = matchphrase('search query')").toString() ); assertEquals(AstDSL.function("matchphrase", unresolvedArg("field", stringLiteral("message")), - unresolvedArg("query", stringLiteral("search query"))), - buildExprAst("message = matchphrase(\"search query\")") + unresolvedArg("query", stringLiteral("search query"))).toString(), + buildExprAst("message = matchphrase(\"search query\")").toString() ); } From 2a1067181a49fcd28fd6435757800c75a2a5ff6b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 09:18:01 -0800 Subject: [PATCH 04/12] Fixed error with single timestamp query (#1244) (#1246) Signed-off-by: vamsi-amazon Signed-off-by: vamsi-amazon (cherry picked from commit ee949ccc128eed78d32b6993948036a93e974bd4) Co-authored-by: vamsi-amazon --- .../SeriesSelectionQueryBuilder.java | 4 ++- .../storage/PrometheusMetricTableTest.java | 31 ++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/SeriesSelectionQueryBuilder.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/SeriesSelectionQueryBuilder.java index baa235aa89..c749c12758 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/SeriesSelectionQueryBuilder.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/SeriesSelectionQueryBuilder.java @@ -37,7 +37,9 @@ public static String build(String metricName, Expression filterCondition) { SeriesSelectionExpressionNodeVisitor seriesSelectionExpressionNodeVisitor = new SeriesSelectionExpressionNodeVisitor(); String selectorQuery = filterCondition.accept(seriesSelectionExpressionNodeVisitor, null); - return metricName + "{" + selectorQuery + "}"; + if (selectorQuery != null) { + return metricName + "{" + selectorQuery + "}"; + } } return metricName; } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java index 7e2de95604..b03b0b9ebc 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java @@ -303,7 +303,7 @@ void testTimeRangeResolverWithOutEndTimeInFilter() { new PrometheusMetricTable(client, "prometheus_http_total_requests"); - //Both endTime and startTime are set. + //Only endTime is set. List finalProjectList = new ArrayList<>(); finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); @@ -724,6 +724,35 @@ void testImplementWithRelationAndFilter() { assertEquals(List.of(VALUE, TIMESTAMP), outputFields); } + @Test + void testImplementWithRelationAndTimestampFilter() { + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Long endTime = new Date(System.currentTimeMillis()).getTime(); + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + LogicalPlan logicalPlan = project(indexScan("prometheus_http_total_requests", + DSL.lte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(endTime)), + ExprCoreType.TIMESTAMP))) + ), finalProjectList, null); + PhysicalPlan physicalPlan = prometheusMetricTable.implement(logicalPlan); + assertTrue(physicalPlan instanceof ProjectOperator); + assertTrue(((ProjectOperator) physicalPlan).getInput() instanceof PrometheusMetricScan); + PrometheusQueryRequest request + = ((PrometheusMetricScan) ((ProjectOperator) physicalPlan).getInput()).getRequest(); + assertEquals((3600 / 250) + "s", request.getStep()); + assertEquals("prometheus_http_total_requests", + request.getPromQl()); + List projectList = ((ProjectOperator) physicalPlan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); + } + @Test void testOptimize() { PrometheusQueryRequest prometheusQueryRequest = new PrometheusQueryRequest(); From 628be978c7421010f5bc85bc358c60ff836cac96 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:05:42 -0800 Subject: [PATCH 05/12] Add Second_Of_Minute Function As An Alias Of The Second Function (#1231) (#1237) Added Testing And Implementation For Second_Of_Minute Function Signed-off-by: GabeFernandez310 Signed-off-by: GabeFernandez310 (cherry picked from commit dce7d0e73e9df173e7afdabd8c7d8a6e65a8a4d3) Co-authored-by: GabeFernandez310 --- .../org/opensearch/sql/expression/DSL.java | 4 + .../expression/datetime/DateTimeFunction.java | 12 ++- .../function/BuiltinFunctionName.java | 1 + .../datetime/DateTimeFunctionTest.java | 76 +++++++++++++++++++ docs/user/dql/functions.rst | 9 +++ .../sql/sql/DateTimeFunctionIT.java | 45 +++++++++++ sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 + .../sql/sql/antlr/SQLSyntaxParserTest.java | 8 ++ 8 files changed, 152 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index dfe380b507..7899e47a71 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -378,6 +378,10 @@ public static FunctionExpression second(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.SECOND, expressions); } + public static FunctionExpression second_of_minute(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.SECOND_OF_MINUTE, expressions); + } + public static FunctionExpression subdate(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.SUBDATE, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 6ee5a79172..a29429fc04 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -9,6 +9,7 @@ import static java.time.temporal.ChronoUnit.DAYS; import static java.time.temporal.ChronoUnit.MINUTES; import static java.time.temporal.ChronoUnit.MONTHS; +import static java.time.temporal.ChronoUnit.SECONDS; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; @@ -128,7 +129,8 @@ public void register(BuiltinFunctionRepository repository) { repository.register(period_add()); repository.register(period_diff()); repository.register(quarter()); - repository.register(second()); + repository.register(second(BuiltinFunctionName.SECOND)); + repository.register(second(BuiltinFunctionName.SECOND_OF_MINUTE)); repository.register(subdate()); repository.register(sysdate()); repository.register(time()); @@ -557,10 +559,11 @@ private DefaultFunctionResolver quarter() { /** * SECOND(STRING/TIME/DATETIME/TIMESTAMP). return the second value for time. */ - private DefaultFunctionResolver second() { - return define(BuiltinFunctionName.SECOND.getName(), + private DefaultFunctionResolver second(BuiltinFunctionName name) { + return define(name.getName(), impl(nullMissingHandling(DateTimeFunction::exprSecond), INTEGER, STRING), impl(nullMissingHandling(DateTimeFunction::exprSecond), INTEGER, TIME), + impl(nullMissingHandling(DateTimeFunction::exprSecond), INTEGER, DATE), impl(nullMissingHandling(DateTimeFunction::exprSecond), INTEGER, DATETIME), impl(nullMissingHandling(DateTimeFunction::exprSecond), INTEGER, TIMESTAMP) ); @@ -1131,7 +1134,8 @@ private ExprValue exprQuarter(ExprValue date) { * @return ExprValue. */ private ExprValue exprSecond(ExprValue time) { - return new ExprIntegerValue(time.timeValue().getSecond()); + return new ExprIntegerValue( + (SECONDS.between(LocalTime.MIN, time.timeValue()) % 60)); } /** diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 50c8682c62..06b46b6888 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -86,6 +86,7 @@ public enum BuiltinFunctionName { PERIOD_DIFF(FunctionName.of("period_diff")), QUARTER(FunctionName.of("quarter")), SECOND(FunctionName.of("second")), + SECOND_OF_MINUTE(FunctionName.of("second_of_minute")), SUBDATE(FunctionName.of("subdate")), TIME(FunctionName.of("time")), TIMEDIFF(FunctionName.of("timediff")), diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 29a0843287..7a486fc8d4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -28,10 +28,14 @@ import com.google.common.collect.ImmutableList; import java.time.LocalDate; import java.util.List; +import java.util.stream.Stream; import lombok.AllArgsConstructor; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprDateValue; @@ -46,6 +50,7 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.env.Environment; @ExtendWith(MockitoExtension.class) @@ -904,6 +909,77 @@ public void second() { assertEquals("second(DATETIME '2020-08-17 01:02:03')", expression.toString()); } + private void secondOfMinuteQuery(FunctionExpression dateExpression, int second, String testExpr) { + assertEquals(INTEGER, dateExpression.type()); + assertEquals(integerValue(second), eval(dateExpression)); + assertEquals(testExpr, dateExpression.toString()); + } + + private static Stream getTestDataForSecondOfMinute() { + return Stream.of( + Arguments.of( + DSL.literal(new ExprTimeValue("01:02:03")), + 3, + "second_of_minute(TIME '01:02:03')"), + Arguments.of( + DSL.literal("01:02:03"), + 3, + "second_of_minute(\"01:02:03\")"), + Arguments.of( + DSL.literal("2020-08-17 01:02:03"), + 3, + "second_of_minute(\"2020-08-17 01:02:03\")"), + Arguments.of( + + DSL.literal(new ExprTimestampValue("2020-08-17 01:02:03")), + 3, + "second_of_minute(TIMESTAMP '2020-08-17 01:02:03')"), + Arguments.of( + + DSL.literal(new ExprDatetimeValue("2020-08-17 01:02:03")), + 3, + "second_of_minute(DATETIME '2020-08-17 01:02:03')") + ); + } + + @ParameterizedTest(name = "{2}") + @MethodSource("getTestDataForSecondOfMinute") + public void secondOfMinute(LiteralExpression arg, int expectedResult, String expectedString) { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + secondOfMinuteQuery(DSL.second_of_minute(arg), expectedResult, expectedString); + } + + private void invalidSecondOfMinuteQuery(String time) { + FunctionExpression expression = DSL.second_of_minute(DSL.literal(new ExprTimeValue(time))); + eval(expression); + } + + @Test + public void secondOfMinuteInvalidArguments() { + when(nullRef.type()).thenReturn(TIME); + when(missingRef.type()).thenReturn(TIME); + + assertAll( + () -> assertEquals(nullValue(), eval(DSL.second_of_minute(nullRef))), + () -> assertEquals(missingValue(), eval(DSL.second_of_minute(missingRef))), + //Invalid Seconds + () -> assertThrows(SemanticCheckException.class, + () -> invalidSecondOfMinuteQuery("12:23:61")), + //Invalid Minutes + () -> assertThrows(SemanticCheckException.class, + () -> invalidSecondOfMinuteQuery("12:61:34")), + //Invalid Hours + () -> assertThrows(SemanticCheckException.class, + () -> invalidSecondOfMinuteQuery("25:23:34")), + //incorrect format + () -> assertThrows(SemanticCheckException.class, + () -> invalidSecondOfMinuteQuery("asdfasdf")) + ); + } + + @Test public void subdate() { FunctionExpression expr = DSL.subdate(DSL.date(DSL.literal("2020-08-26")), DSL.literal(7)); diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 1726c92054..f95947a7b5 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1961,6 +1961,7 @@ Description >>>>>>>>>>> Usage: second(time) returns the second for time, in the range 0 to 59. +The function `second_of_minute`_ is provided as an alias Argument type: STRING/TIME/DATETIME/TIMESTAMP @@ -1976,6 +1977,14 @@ Example:: | 3 | +-----------------------------+ + os> SELECT SECOND_OF_MINUTE(time('01:02:03')) + fetched rows / total rows = 1/1 + +--------------------------------------+ + | SECOND_OF_MINUTE(time('01:02:03')) | + |--------------------------------------| + | 3 | + +--------------------------------------+ + SUBDATE ------- diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index e135f15523..62bb033af3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -493,6 +493,51 @@ public void testSecond() throws IOException { verifyDataRows(result, rows(0)); } + @Test + public void testSecondOfMinute() throws IOException { + JSONObject result = executeQuery("select second_of_minute(timestamp('2020-09-16 17:30:00'))"); + verifySchema(result, schema("second_of_minute(timestamp('2020-09-16 17:30:00'))", null, "integer")); + verifyDataRows(result, rows(0)); + + result = executeQuery("select second_of_minute(time('17:30:00'))"); + verifySchema(result, schema("second_of_minute(time('17:30:00'))", null, "integer")); + verifyDataRows(result, rows(0)); + + result = executeQuery("select second_of_minute('2020-09-16 17:30:00')"); + verifySchema(result, schema("second_of_minute('2020-09-16 17:30:00')", null, "integer")); + verifyDataRows(result, rows(0)); + + result = executeQuery("select second_of_minute('17:30:00')"); + verifySchema(result, schema("second_of_minute('17:30:00')", null, "integer")); + verifyDataRows(result, rows(0)); + } + + @Test + public void testSecondFunctionAliasesReturnTheSameResults() throws IOException { + JSONObject result1 = executeQuery("SELECT second('2022-11-22 12:23:34')"); + JSONObject result2 = executeQuery("SELECT second_of_minute('2022-11-22 12:23:34')"); + verifyDataRows(result1, rows(34)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT second(datetime(CAST(time0 AS STRING))) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT second_of_minute(datetime(CAST(time0 AS STRING))) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT second(CAST(time0 AS STRING)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT second_of_minute(CAST(time0 AS STRING)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT second(CAST(datetime0 AS timestamp)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT second_of_minute(CAST(datetime0 AS timestamp)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + } + @Test public void testSubDate() throws IOException { JSONObject result = diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index f10b6af186..cc8f5e0a8d 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -455,6 +455,7 @@ dateTimeFunctionName | PERIOD_DIFF | QUARTER | SECOND + | SECOND_OF_MINUTE | SUBDATE | SYSDATE | TIME diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index d1a8f29a2c..487f300dd7 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -268,6 +268,14 @@ public void can_parse_multi_match_relevance_function() { + "operator='AND', tie_breaker=0.3, type = \"most_fields\", fuzziness = \"AUTO\")")); } + @Test + public void can_parse_second_functions() { + assertNotNull(parser.parse("SELECT second('12:23:34')")); + assertNotNull(parser.parse("SELECT second_of_minute('2022-11-18')")); + assertNotNull(parser.parse("SELECT second('2022-11-18 12:23:34')")); + assertNotNull(parser.parse("SELECT second_of_minute('2022-11-18 12:23:34')")); + } + @Test public void can_parse_simple_query_string_relevance_function() { assertNotNull(parser.parse( From 571dcd1a230e3484aafdd9d2073f1ceecc287078 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 11:15:10 -0800 Subject: [PATCH 06/12] Add functions `ADDTIME` and `SUBTIME`. (#132) (#1194) (#1252) * Add functions `ADDTIME` and `SUBTIME`. (#132) Signed-off-by: Yury-Fridlyand (cherry picked from commit 7630f87335d62f7fd5ec3eb5c49697c97f94296d) Co-authored-by: Yury-Fridlyand --- .../expression/datetime/DateTimeFunction.java | 140 ++++++++++++++++++ .../function/BuiltinFunctionName.java | 2 + .../opensearch/sql/utils/DateTimeUtils.java | 11 ++ .../datetime/AddTimeAndSubTimeTest.java | 127 ++++++++++++++++ .../expression/datetime/DateTimeTestBase.java | 34 +++-- docs/user/dql/functions.rst | 122 +++++++++++++++ docs/user/ppl/functions/datetime.rst | 125 +++++++++++++++- .../sql/ppl/DateTimeFunctionIT.java | 36 +++++ .../sql/sql/DateTimeFunctionIT.java | 33 +++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 + sql/src/main/antlr/OpenSearchSQLLexer.g4 | 2 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 2 + 13 files changed, 626 insertions(+), 12 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/AddTimeAndSubTimeTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index a29429fc04..ade6170102 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -30,6 +30,7 @@ import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_SHORT_YEAR; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_STRICT_WITH_TZ; import static org.opensearch.sql.utils.DateTimeUtils.extractDate; +import static org.opensearch.sql.utils.DateTimeUtils.extractDateTime; import java.math.BigDecimal; import java.math.RoundingMode; @@ -95,6 +96,7 @@ public class DateTimeFunction { */ public void register(BuiltinFunctionRepository repository) { repository.register(adddate()); + repository.register(addtime()); repository.register(convert_tz()); repository.register(curtime()); repository.register(curdate()); @@ -132,6 +134,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(second(BuiltinFunctionName.SECOND)); repository.register(second(BuiltinFunctionName.SECOND_OF_MINUTE)); repository.register(subdate()); + repository.register(subtime()); repository.register(sysdate()); repository.register(time()); repository.register(time_to_sec()); @@ -247,6 +250,52 @@ private DefaultFunctionResolver adddate() { return add_date(BuiltinFunctionName.ADDDATE.getName()); } + /** + * Adds expr2 to expr1 and returns the result. + * (TIME, TIME/DATE/DATETIME/TIMESTAMP) -> TIME + * (DATE/DATETIME/TIMESTAMP, TIME/DATE/DATETIME/TIMESTAMP) -> DATETIME + * TODO: MySQL has these signatures too + * (STRING, STRING/TIME) -> STRING // second arg - string with time only + * (x, STRING) -> NULL // second arg - string with timestamp + * (x, STRING/DATE) -> x // second arg - string with date only + */ + private DefaultFunctionResolver addtime() { + return define(BuiltinFunctionName.ADDTIME.getName(), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + TIME, TIME, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + TIME, TIME, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + TIME, TIME, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + TIME, TIME, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, DATETIME, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, DATETIME, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, DATETIME, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, DATETIME, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, DATE, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, DATE, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, DATE, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, DATE, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, TIMESTAMP, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, TIMESTAMP, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, TIMESTAMP, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + DATETIME, TIMESTAMP, TIMESTAMP) + ); + } + /** * Converts date/time from a specified timezone to another specified timezone. * The supported signatures: @@ -573,6 +622,52 @@ private DefaultFunctionResolver subdate() { return sub_date(BuiltinFunctionName.SUBDATE.getName()); } + /** + * Subtracts expr2 from expr1 and returns the result. + * (TIME, TIME/DATE/DATETIME/TIMESTAMP) -> TIME + * (DATE/DATETIME/TIMESTAMP, TIME/DATE/DATETIME/TIMESTAMP) -> DATETIME + * TODO: MySQL has these signatures too + * (STRING, STRING/TIME) -> STRING // second arg - string with time only + * (x, STRING) -> NULL // second arg - string with timestamp + * (x, STRING/DATE) -> x // second arg - string with date only + */ + private DefaultFunctionResolver subtime() { + return define(BuiltinFunctionName.SUBTIME.getName(), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + TIME, TIME, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + TIME, TIME, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + TIME, TIME, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + TIME, TIME, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, DATETIME, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, DATETIME, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, DATETIME, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, DATETIME, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, DATE, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, DATE, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, DATE, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, DATE, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, TIMESTAMP, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, TIMESTAMP, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, TIMESTAMP, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + DATETIME, TIMESTAMP, TIMESTAMP) + ); + } + /** * Extracts the time part of a date and time value. * Also to construct a time type. The supported signatures: @@ -752,6 +847,39 @@ private ExprValue exprAddDateDays(ExprValue date, ExprValue days) { : exprValue); } + /** + * Adds or subtracts time to/from date and returns the result. + * + * @param functionProperties A FunctionProperties object. + * @param temporal A Date/Time/Datetime/Timestamp value to change. + * @param temporalDelta A Date/Time/Datetime/Timestamp object to add/subtract time from. + * @param isAdd A flag: true to add, false to subtract. + * @return A value calculated. + */ + private ExprValue exprApplyTime(FunctionProperties functionProperties, + ExprValue temporal, ExprValue temporalDelta, Boolean isAdd) { + var interval = Duration.between(LocalTime.MIN, temporalDelta.timeValue()); + var result = isAdd + ? extractDateTime(temporal, functionProperties).plus(interval) + : extractDateTime(temporal, functionProperties).minus(interval); + return temporal.type() == TIME + ? new ExprTimeValue(result.toLocalTime()) + : new ExprDatetimeValue(result); + } + + /** + * Adds time to date and returns the result. + * + * @param functionProperties A FunctionProperties object. + * @param temporal A Date/Time/Datetime/Timestamp value to change. + * @param temporalDelta A Date/Time/Datetime/Timestamp object to add time from. + * @return A value calculated. + */ + private ExprValue exprAddTime(FunctionProperties functionProperties, + ExprValue temporal, ExprValue temporalDelta) { + return exprApplyTime(functionProperties, temporal, temporalDelta, true); + } + /** * CONVERT_TZ function implementation for ExprValue. * Returns null for time zones outside of +13:00 and -12:00. @@ -1164,6 +1292,18 @@ private ExprValue exprSubDateInterval(ExprValue date, ExprValue expr) { : exprValue); } + /** + * Subtracts expr2 from expr1 and returns the result. + * + * @param temporal A Date/Time/Datetime/Timestamp value to change. + * @param temporalDelta A Date/Time/Datetime/Timestamp to subtract time from. + * @return A value calculated. + */ + private ExprValue exprSubTime(FunctionProperties functionProperties, + ExprValue temporal, ExprValue temporalDelta) { + return exprApplyTime(functionProperties, temporal, temporalDelta, false); + } + /** * Time implementation for ExprValue. * diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 06b46b6888..cdc0b4e30d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -59,6 +59,7 @@ public enum BuiltinFunctionName { * Date and Time Functions. */ ADDDATE(FunctionName.of("adddate")), + ADDTIME(FunctionName.of("addtime")), CONVERT_TZ(FunctionName.of("convert_tz")), DATE(FunctionName.of("date")), DATEDIFF(FunctionName.of("datediff")), @@ -88,6 +89,7 @@ public enum BuiltinFunctionName { SECOND(FunctionName.of("second")), SECOND_OF_MINUTE(FunctionName.of("second_of_minute")), SUBDATE(FunctionName.of("subdate")), + SUBTIME(FunctionName.of("subtime")), TIME(FunctionName.of("time")), TIMEDIFF(FunctionName.of("timediff")), TIME_TO_SEC(FunctionName.of("time_to_sec")), diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java index 28bb4e6918..06d40ea6aa 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java @@ -130,6 +130,17 @@ public Boolean isValidMySqlTimeZoneId(ZoneId zone) { || passedTzValidator.isEqual(minTzValidator)); } + /** + * Extracts LocalDateTime from a datetime ExprValue. + * Uses `FunctionProperties` for `ExprTimeValue`. + */ + public static LocalDateTime extractDateTime(ExprValue value, + FunctionProperties functionProperties) { + return value instanceof ExprTimeValue + ? ((ExprTimeValue) value).datetimeValue(functionProperties) + : value.datetimeValue(); + } + /** * Extracts LocalDate from a datetime ExprValue. * Uses `FunctionProperties` for `ExprTimeValue`. diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/AddTimeAndSubTimeTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/AddTimeAndSubTimeTest.java new file mode 100644 index 0000000000..e917e2ee62 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/AddTimeAndSubTimeTest.java @@ -0,0 +1,127 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.datetime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.temporal.Temporal; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class AddTimeAndSubTimeTest extends DateTimeTestBase { + + @Test + // (TIME, TIME/DATE/DATETIME/TIMESTAMP) -> TIME + public void return_time_when_first_arg_is_time() { + var res = addtime(LocalTime.of(21, 0), LocalTime.of(0, 5)); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(21, 5), res.timeValue()); + + res = subtime(LocalTime.of(21, 0), LocalTime.of(0, 5)); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(20, 55), res.timeValue()); + + res = addtime(LocalTime.of(12, 20), Instant.ofEpochSecond(42)); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(12, 20, 42), res.timeValue()); + + res = subtime(LocalTime.of(10, 0), Instant.ofEpochSecond(42)); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(9, 59, 18), res.timeValue()); + + res = addtime(LocalTime.of(2, 3, 4), LocalDateTime.of(1961, 4, 12, 9, 7)); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(11, 10, 4), res.timeValue()); + + res = subtime(LocalTime.of(12, 3, 4), LocalDateTime.of(1961, 4, 12, 9, 7)); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(2, 56, 4), res.timeValue()); + + res = addtime(LocalTime.of(9, 7), LocalDate.now()); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(9, 7), res.timeValue()); + + res = subtime(LocalTime.of(9, 7), LocalDate.of(1961, 4, 12)); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(9, 7), res.timeValue()); + } + + @Test + public void time_limited_by_24_hours() { + var res = addtime(LocalTime.of(21, 0), LocalTime.of(14, 5)); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(11, 5), res.timeValue()); + + res = subtime(LocalTime.of(14, 0), LocalTime.of(21, 5)); + assertEquals(TIME, res.type()); + assertEquals(LocalTime.of(16, 55), res.timeValue()); + } + + // Function signature is: + // (DATE/DATETIME/TIMESTAMP, TIME/DATE/DATETIME/TIMESTAMP) -> DATETIME + private static Stream getTestData() { + return Stream.of( + // DATETIME and TIME/DATE/DATETIME/TIMESTAMP + Arguments.of(LocalDateTime.of(1961, 4, 12, 9, 7), LocalTime.of(1, 48), + LocalDateTime.of(1961, 4, 12, 10, 55), LocalDateTime.of(1961, 4, 12, 7, 19)), + Arguments.of(LocalDateTime.of(1961, 4, 12, 9, 7), LocalDate.of(2000, 1, 1), + LocalDateTime.of(1961, 4, 12, 9, 7), LocalDateTime.of(1961, 4, 12, 9, 7)), + Arguments.of(LocalDateTime.of(1961, 4, 12, 9, 7), LocalDateTime.of(1235, 5, 6, 1, 48), + LocalDateTime.of(1961, 4, 12, 10, 55), LocalDateTime.of(1961, 4, 12, 7, 19)), + Arguments.of(LocalDateTime.of(1961, 4, 12, 9, 7), Instant.ofEpochSecond(42), + LocalDateTime.of(1961, 4, 12, 9, 7, 42), LocalDateTime.of(1961, 4, 12, 9, 6, 18)), + // DATE and TIME/DATE/DATETIME/TIMESTAMP + Arguments.of(LocalDate.of(1961, 4, 12), LocalTime.of(9, 7), + LocalDateTime.of(1961, 4, 12, 9, 7), LocalDateTime.of(1961, 4, 11, 14, 53)), + Arguments.of(LocalDate.of(1961, 4, 12), LocalDate.of(2000, 1, 1), + LocalDateTime.of(1961, 4, 12, 0, 0), LocalDateTime.of(1961, 4, 12, 0, 0)), + Arguments.of(LocalDate.of(1961, 4, 12), LocalDateTime.of(1235, 5, 6, 1, 48), + LocalDateTime.of(1961, 4, 12, 1, 48), LocalDateTime.of(1961, 4, 11, 22, 12)), + Arguments.of(LocalDate.of(1961, 4, 12), Instant.ofEpochSecond(42), + LocalDateTime.of(1961, 4, 12, 0, 0, 42), LocalDateTime.of(1961, 4, 11, 23, 59, 18)), + // TIMESTAMP and TIME/DATE/DATETIME/TIMESTAMP + Arguments.of(Instant.ofEpochSecond(42), LocalTime.of(9, 7), + LocalDateTime.of(1970, 1, 1, 9, 7, 42), LocalDateTime.of(1969, 12, 31, 14, 53, 42)), + Arguments.of(Instant.ofEpochSecond(42), LocalDate.of(1961, 4, 12), + LocalDateTime.of(1970, 1, 1, 0, 0, 42), LocalDateTime.of(1970, 1, 1, 0, 0, 42)), + Arguments.of(Instant.ofEpochSecond(42), LocalDateTime.of(1961, 4, 12, 9, 7), + LocalDateTime.of(1970, 1, 1, 9, 7, 42), LocalDateTime.of(1969, 12, 31, 14, 53, 42)), + Arguments.of(Instant.ofEpochSecond(42), Instant.ofEpochMilli(42), + LocalDateTime.of(1970, 1, 1, 0, 0, 42, 42000000), + LocalDateTime.of(1970, 1, 1, 0, 0, 41, 958000000)) + ); + } + + /** + * Check that `ADDTIME` and `SUBTIME` functions result value and type. + * @param arg1 First argument. + * @param arg2 Second argument. + * @param addTimeExpectedResult Expected result for `ADDTIME`. + * @param subTimeExpectedResult Expected result for `SUBTIME`. + */ + @ParameterizedTest + @MethodSource("getTestData") + public void return_datetime_when_first_arg_is_not_time(Temporal arg1, Temporal arg2, + LocalDateTime addTimeExpectedResult, + LocalDateTime subTimeExpectedResult) { + var res = addtime(arg1, arg2); + assertEquals(DATETIME, res.type()); + assertEquals(addTimeExpectedResult, res.datetimeValue()); + + res = subtime(arg1, arg2); + assertEquals(DATETIME, res.type()); + assertEquals(subTimeExpectedResult, res.datetimeValue()); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java index 7517b27944..d8829ea41a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -14,10 +14,6 @@ import java.time.LocalTime; import java.time.temporal.Temporal; import java.util.List; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; import org.opensearch.sql.data.model.ExprMissingValue; @@ -29,26 +25,31 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.FunctionProperties; -@ExtendWith(MockitoExtension.class) public class DateTimeTestBase extends ExpressionTestBase { protected final BuiltinFunctionRepository functionRepository = BuiltinFunctionRepository.getInstance(); - @Mock - protected Environment env; - protected Expression nullRef = DSL.literal(ExprNullValue.of()); protected Expression missingRef = DSL.literal(ExprMissingValue.of()); protected ExprValue eval(Expression expression) { - return expression.valueOf(env); + return expression.valueOf(); + } + + protected FunctionExpression addtime(Expression date, Expression interval) { + return (FunctionExpression) functionRepository.compile( + functionProperties, + BuiltinFunctionName.ADDTIME.getName(), List.of(date, interval)); + } + + protected ExprValue addtime(Temporal first, Temporal second) { + return addtime(DSL.literal(fromObjectValue(first)), DSL.literal(fromObjectValue(second))) + .valueOf(null); } protected FunctionExpression datediff(Expression first, Expression second) { @@ -137,6 +138,17 @@ protected Integer period_diff(Integer first, Integer second) { .valueOf().integerValue(); } + protected FunctionExpression subtime(Expression date, Expression interval) { + return (FunctionExpression) functionRepository.compile( + functionProperties, + BuiltinFunctionName.SUBTIME.getName(), List.of(date, interval)); + } + + protected ExprValue subtime(Temporal first, Temporal second) { + return subtime(DSL.literal(fromObjectValue(first)), DSL.literal(fromObjectValue(second))) + .valueOf(null); + } + protected FunctionExpression timediff(Expression first, Expression second) { return (FunctionExpression) functionRepository.compile( functionProperties, diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index f95947a7b5..86d18f1c2f 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -873,6 +873,67 @@ Example:: +------------------------------------------------+----------------------------------+------------------------------------------------+ +ADDTIME +------- + +Description +>>>>>>>>>>> + +Usage: addtime(expr1, expr2) adds expr2 to expr1 and returns the result. If argument is TIME, today's date is used; if argument is DATE, time at midnight is used. + +Argument type: DATE/DATETIME/TIMESTAMP/TIME, DATE/DATETIME/TIMESTAMP/TIME + +Return type map: + +(DATE/DATETIME/TIMESTAMP, DATE/DATETIME/TIMESTAMP/TIME) -> DATETIME + +(TIME, DATE/DATETIME/TIMESTAMP/TIME) -> TIME + +Antonyms: `SUBTIME`_ + +Example:: + + os> SELECT ADDTIME(DATE('2008-12-12'), DATE('2008-11-15')) AS `'2008-12-12' + 0 ` + fetched rows / total rows = 1/1 + +---------------------+ + | '2008-12-12' + 0 | + |---------------------| + | 2008-12-12 00:00:00 | + +---------------------+ + + os> SELECT ADDTIME(TIME('23:59:59'), DATE('2004-01-01')) AS `'23:59:59' + 0` + fetched rows / total rows = 1/1 + +------------------+ + | '23:59:59' + 0 | + |------------------| + | 23:59:59 | + +------------------+ + + os> SELECT ADDTIME(DATE('2004-01-01'), TIME('23:59:59')) AS `'2004-01-01' + '23:59:59'` + fetched rows / total rows = 1/1 + +-----------------------------+ + | '2004-01-01' + '23:59:59' | + |-----------------------------| + | 2004-01-01 23:59:59 | + +-----------------------------+ + + os> SELECT ADDTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' + '00:05:42'` + fetched rows / total rows = 1/1 + +---------------------------+ + | '10:20:30' + '00:05:42' | + |---------------------------| + | 10:26:12 | + +---------------------------+ + + os> SELECT ADDTIME(TIMESTAMP('2007-02-28 10:20:30'), DATETIME('2002-03-04 20:40:50')) AS `'2007-02-28 10:20:30' + '20:40:50'` + fetched rows / total rows = 1/1 + +--------------------------------------+ + | '2007-02-28 10:20:30' + '20:40:50' | + |--------------------------------------| + | 2007-03-01 07:01:20 | + +--------------------------------------+ + + CONVERT_TZ ---------- @@ -2017,6 +2078,67 @@ Example:: +------------------------------------------------+----------------------------------+------------------------------------------------+ +SUBTIME +------- + +Description +>>>>>>>>>>> + +Usage: subtime(expr1, expr2) subtracts expr2 from expr1 and returns the result. If argument is TIME, today's date is used; if argument is DATE, time at midnight is used. + +Argument type: DATE/DATETIME/TIMESTAMP/TIME, DATE/DATETIME/TIMESTAMP/TIME + +Return type map: + +(DATE/DATETIME/TIMESTAMP, DATE/DATETIME/TIMESTAMP/TIME) -> DATETIME + +(TIME, DATE/DATETIME/TIMESTAMP/TIME) -> TIME + +Antonyms: `ADDTIME`_ + +Example:: + + os> SELECT SUBTIME(DATE('2008-12-12'), DATE('2008-11-15')) AS `'2008-12-12' - 0` + fetched rows / total rows = 1/1 + +---------------------+ + | '2008-12-12' - 0 | + |---------------------| + | 2008-12-12 00:00:00 | + +---------------------+ + + os> SELECT SUBTIME(TIME('23:59:59'), DATE('2004-01-01')) AS `'23:59:59' - 0` + fetched rows / total rows = 1/1 + +------------------+ + | '23:59:59' - 0 | + |------------------| + | 23:59:59 | + +------------------+ + + os> SELECT SUBTIME(DATE('2004-01-01'), TIME('23:59:59')) AS `'2004-01-01' - '23:59:59'` + fetched rows / total rows = 1/1 + +-----------------------------+ + | '2004-01-01' - '23:59:59' | + |-----------------------------| + | 2003-12-31 00:00:01 | + +-----------------------------+ + + os> SELECT SUBTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' - '00:05:42'` + fetched rows / total rows = 1/1 + +---------------------------+ + | '10:20:30' - '00:05:42' | + |---------------------------| + | 10:14:48 | + +---------------------------+ + + os> SELECT SUBTIME(TIMESTAMP('2007-03-01 10:20:30'), DATETIME('2002-03-04 20:40:50')) AS `'2007-03-01 10:20:30' - '20:40:50'` + fetched rows / total rows = 1/1 + +--------------------------------------+ + | '2007-03-01 10:20:30' - '20:40:50' | + |--------------------------------------| + | 2007-02-28 13:39:40 | + +--------------------------------------+ + + SYSDATE ------- diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 60a8660c7a..5e54d8e80f 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -38,8 +38,70 @@ Example:: | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | +------------------------------------------------+----------------------------------+------------------------------------------------+ + +ADDTIME +------- + +Description +>>>>>>>>>>> + +Usage: addtime(expr1, expr2) adds expr2 to expr1 and returns the result. If argument is TIME, today's date is used; if argument is DATE, time at midnight is used. + +Argument type: DATE/DATETIME/TIMESTAMP/TIME, DATE/DATETIME/TIMESTAMP/TIME + +Return type map: + +(DATE/DATETIME/TIMESTAMP, DATE/DATETIME/TIMESTAMP/TIME) -> DATETIME + +(TIME, DATE/DATETIME/TIMESTAMP/TIME) -> TIME + +Antonyms: `SUBTIME`_ + +Example:: + + os> source=people | eval `'2008-12-12' + 0` = ADDTIME(DATE('2008-12-12'), DATE('2008-11-15')) | fields `'2008-12-12' + 0` + fetched rows / total rows = 1/1 + +---------------------+ + | '2008-12-12' + 0 | + |---------------------| + | 2008-12-12 00:00:00 | + +---------------------+ + + os> source=people | eval `'23:59:59' + 0` = ADDTIME(TIME('23:59:59'), DATE('2004-01-01')) | fields `'23:59:59' + 0` + fetched rows / total rows = 1/1 + +------------------+ + | '23:59:59' + 0 | + |------------------| + | 23:59:59 | + +------------------+ + + os> source=people | eval `'2004-01-01' + '23:59:59'` = ADDTIME(DATE('2004-01-01'), TIME('23:59:59')) | fields `'2004-01-01' + '23:59:59'` + fetched rows / total rows = 1/1 + +-----------------------------+ + | '2004-01-01' + '23:59:59' | + |-----------------------------| + | 2004-01-01 23:59:59 | + +-----------------------------+ + + os> source=people | eval `'10:20:30' + '00:05:42'` = ADDTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' + '00:05:42'` + fetched rows / total rows = 1/1 + +---------------------------+ + | '10:20:30' + '00:05:42' | + |---------------------------| + | 10:26:12 | + +---------------------------+ + + os> source=people | eval `'2007-02-28 10:20:30' + '20:40:50'` = ADDTIME(TIMESTAMP('2007-02-28 10:20:30'), DATETIME('2002-03-04 20:40:50')) | fields `'2007-02-28 10:20:30' + '20:40:50'` + fetched rows / total rows = 1/1 + +--------------------------------------+ + | '2007-02-28 10:20:30' + '20:40:50' | + |--------------------------------------| + | 2007-03-01 07:01:20 | + +--------------------------------------+ + + CONVERT_TZ ----- +---------- Description >>>>>>>>>>> @@ -1102,6 +1164,67 @@ Example:: +------------------------------------------------+----------------------------------+------------------------------------------------+ +SUBTIME +------- + +Description +>>>>>>>>>>> + +Usage: subtime(expr1, expr2) subtracts expr2 from expr1 and returns the result. If argument is TIME, today's date is used; if argument is DATE, time at midnight is used. + +Argument type: DATE/DATETIME/TIMESTAMP/TIME, DATE/DATETIME/TIMESTAMP/TIME + +Return type map: + +(DATE/DATETIME/TIMESTAMP, DATE/DATETIME/TIMESTAMP/TIME) -> DATETIME + +(TIME, DATE/DATETIME/TIMESTAMP/TIME) -> TIME + +Antonyms: `ADDTIME`_ + +Example:: + + os> source=people | eval `'2008-12-12' - 0` = SUBTIME(DATE('2008-12-12'), DATE('2008-11-15')) | fields `'2008-12-12' - 0` + fetched rows / total rows = 1/1 + +---------------------+ + | '2008-12-12' - 0 | + |---------------------| + | 2008-12-12 00:00:00 | + +---------------------+ + + os> source=people | eval `'23:59:59' - 0` = SUBTIME(TIME('23:59:59'), DATE('2004-01-01')) | fields `'23:59:59' - 0` + fetched rows / total rows = 1/1 + +------------------+ + | '23:59:59' - 0 | + |------------------| + | 23:59:59 | + +------------------+ + + os> source=people | eval `'2004-01-01' - '23:59:59'` = SUBTIME(DATE('2004-01-01'), TIME('23:59:59')) | fields `'2004-01-01' - '23:59:59'` + fetched rows / total rows = 1/1 + +-----------------------------+ + | '2004-01-01' - '23:59:59' | + |-----------------------------| + | 2003-12-31 00:00:01 | + +-----------------------------+ + + os> source=people | eval `'10:20:30' - '00:05:42'` = SUBTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' - '00:05:42'` + fetched rows / total rows = 1/1 + +---------------------------+ + | '10:20:30' - '00:05:42' | + |---------------------------| + | 10:14:48 | + +---------------------------+ + + os> source=people | eval `'2007-03-01 10:20:30' - '20:40:50'` = SUBTIME(TIMESTAMP('2007-03-01 10:20:30'), DATETIME('2002-03-04 20:40:50')) | fields `'2007-03-01 10:20:30' - '20:40:50'` + fetched rows / total rows = 1/1 + +--------------------------------------+ + | '2007-03-01 10:20:30' - '20:40:50' | + |--------------------------------------| + | 2007-02-28 13:39:40 | + +--------------------------------------+ + + SYSDATE ------- diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 9f59caefc6..23f2df69c8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -865,6 +865,42 @@ public void testNowLikeFunctions() throws IOException { TimeZone.setDefault(testTz); } + @Test + public void testAddTime() throws IOException { + var result = executeQuery(String.format("source=%s | eval" + + " `'2008-12-12' + 0` = ADDTIME(DATE('2008-12-12'), DATE('2008-11-15'))," + + " `'23:59:59' + 0` = ADDTIME(TIME('23:59:59'), DATE('2004-01-01'))," + + " `'2004-01-01' + '23:59:59'` = ADDTIME(DATE('2004-01-01'), TIME('23:59:59'))," + + " `'10:20:30' + '00:05:42'` = ADDTIME(TIME('10:20:30'), TIME('00:05:42'))," + + " `'15:42:13' + '09:07:00'` = ADDTIME(TIMESTAMP('1999-12-31 15:42:13'), DATETIME('1961-04-12 09:07:00'))" + + " | fields `'2008-12-12' + 0`, `'23:59:59' + 0`, `'2004-01-01' + '23:59:59'`, `'10:20:30' + '00:05:42'`, `'15:42:13' + '09:07:00'`", TEST_INDEX_DATE)); + verifySchema(result, + schema("'2008-12-12' + 0", null, "datetime"), + schema("'23:59:59' + 0", null, "time"), + schema("'2004-01-01' + '23:59:59'", null, "datetime"), + schema("'10:20:30' + '00:05:42'", null, "time"), + schema("'15:42:13' + '09:07:00'", null, "datetime")); + verifySome(result.getJSONArray("datarows"), rows("2008-12-12 00:00:00", "23:59:59", "2004-01-01 23:59:59", "10:26:12", "2000-01-01 00:49:13")); + } + + @Test + public void testSubTime() throws IOException { + var result = executeQuery(String.format("source=%s | eval" + + " `'2008-12-12' - 0` = SUBTIME(DATE('2008-12-12'), DATE('2008-11-15'))," + + " `'23:59:59' - 0` = SUBTIME(TIME('23:59:59'), DATE('2004-01-01'))," + + " `'2004-01-01' - '23:59:59'` = SUBTIME(DATE('2004-01-01'), TIME('23:59:59'))," + + " `'10:20:30' - '00:05:42'` = SUBTIME(TIME('10:20:30'), TIME('00:05:42'))," + + " `'15:42:13' - '09:07:00'` = SUBTIME(TIMESTAMP('1999-12-31 15:42:13'), DATETIME('1961-04-12 09:07:00'))" + + " | fields `'2008-12-12' - 0`, `'23:59:59' - 0`, `'2004-01-01' - '23:59:59'`, `'10:20:30' - '00:05:42'`, `'15:42:13' - '09:07:00'`", TEST_INDEX_DATE)); + verifySchema(result, + schema("'2008-12-12' - 0", null, "datetime"), + schema("'23:59:59' - 0", null, "time"), + schema("'2004-01-01' - '23:59:59'", null, "datetime"), + schema("'10:20:30' - '00:05:42'", null, "time"), + schema("'15:42:13' - '09:07:00'", null, "datetime")); + verifySome(result.getJSONArray("datarows"), rows("2008-12-12 00:00:00", "23:59:59", "2003-12-31 00:00:01", "10:14:48", "1999-12-31 06:35:13")); + } + @Test public void testFromUnixTime() throws IOException { var result = executeQuery(String.format( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 62bb033af3..389c976f5c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -949,6 +949,39 @@ public void testPeriodDiff() throws IOException { verifyDataRows(result, rows(11, -25)); } + public void testAddTime() throws IOException { + var result = executeQuery("SELECT" + + " ADDTIME(DATE('2008-12-12'), DATE('2008-11-15')) AS `'2008-12-12' + 0`," + + " ADDTIME(TIME('23:59:59'), DATE('2004-01-01')) AS `'23:59:59' + 0`," + + " ADDTIME(DATE('2004-01-01'), TIME('23:59:59')) AS `'2004-01-01' + '23:59:59'`," + + " ADDTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' + '00:05:42'`," + + " ADDTIME(TIMESTAMP('1999-12-31 15:42:13'), DATETIME('1961-04-12 09:07:00')) AS `'15:42:13' + '09:07:00'`"); + verifySchema(result, + schema("ADDTIME(DATE('2008-12-12'), DATE('2008-11-15'))", "'2008-12-12' + 0", "datetime"), + schema("ADDTIME(TIME('23:59:59'), DATE('2004-01-01'))", "'23:59:59' + 0", "time"), + schema("ADDTIME(DATE('2004-01-01'), TIME('23:59:59'))", "'2004-01-01' + '23:59:59'", "datetime"), + schema("ADDTIME(TIME('10:20:30'), TIME('00:05:42'))", "'10:20:30' + '00:05:42'", "time"), + schema("ADDTIME(TIMESTAMP('1999-12-31 15:42:13'), DATETIME('1961-04-12 09:07:00'))", "'15:42:13' + '09:07:00'", "datetime")); + verifyDataRows(result, rows("2008-12-12 00:00:00", "23:59:59", "2004-01-01 23:59:59", "10:26:12", "2000-01-01 00:49:13")); + } + + @Test + public void testSubTime() throws IOException { + var result = executeQuery("SELECT" + + " SUBTIME(DATE('2008-12-12'), DATE('2008-11-15')) AS `'2008-12-12' - 0`," + + " SUBTIME(TIME('23:59:59'), DATE('2004-01-01')) AS `'23:59:59' - 0`," + + " SUBTIME(DATE('2004-01-01'), TIME('23:59:59')) AS `'2004-01-01' - '23:59:59'`," + + " SUBTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' - '00:05:42'`," + + " SUBTIME(TIMESTAMP('1999-12-31 15:42:13'), DATETIME('1961-04-12 09:07:00')) AS `'15:42:13' - '09:07:00'`"); + verifySchema(result, + schema("SUBTIME(DATE('2008-12-12'), DATE('2008-11-15'))", "'2008-12-12' - 0", "datetime"), + schema("SUBTIME(TIME('23:59:59'), DATE('2004-01-01'))", "'23:59:59' - 0", "time"), + schema("SUBTIME(DATE('2004-01-01'), TIME('23:59:59'))", "'2004-01-01' - '23:59:59'", "datetime"), + schema("SUBTIME(TIME('10:20:30'), TIME('00:05:42'))", "'10:20:30' - '00:05:42'", "time"), + schema("SUBTIME(TIMESTAMP('1999-12-31 15:42:13'), DATETIME('1961-04-12 09:07:00'))", "'15:42:13' - '09:07:00'", "datetime")); + verifyDataRows(result, rows("2008-12-12 00:00:00", "23:59:59", "2003-12-31 00:00:01", "10:14:48", "1999-12-31 06:35:13")); + } + public void testDateDiff() throws IOException { var result = executeQuery("SELECT" + " DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59')) AS `'2000-01-02' - '2000-01-01'`," diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 778d9fb3f9..12c24bd531 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -243,6 +243,7 @@ CURRENT_DATE: 'CURRENT_DATE'; CURRENT_TIME: 'CURRENT_TIME'; CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; CURTIME: 'CURTIME'; +ADDTIME: 'ADDTIME'; DATE: 'DATE'; DATE_ADD: 'DATE_ADD'; DATE_FORMAT: 'DATE_FORMAT'; @@ -263,6 +264,7 @@ NOW: 'NOW'; PERIOD_ADD: 'PERIOD_ADD'; PERIOD_DIFF: 'PERIOD_DIFF'; SUBDATE: 'SUBDATE'; +SUBTIME: 'SUBTIME'; SYSDATE: 'SYSDATE'; TIME: 'TIME'; TIME_TO_SEC: 'TIME_TO_SEC'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index cb9bfae32f..dbb63ef12a 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -433,6 +433,7 @@ trigonometricFunctionName dateAndTimeFunctionBase : ADDDATE + | ADDTIME | CONVERT_TZ | CURRENT_DATE | CURRENT_TIME @@ -467,6 +468,7 @@ dateAndTimeFunctionBase | QUARTER | SECOND | SUBDATE + | SUBTIME | SYSDATE | TIME | TIME_TO_SEC diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 63fd51fc5d..941c1a4c4e 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -176,6 +176,7 @@ TABLES: 'TABLES'; ABS: 'ABS'; ACOS: 'ACOS'; ADD: 'ADD'; +ADDTIME: 'ADDTIME'; ASCII: 'ASCII'; ASIN: 'ASIN'; ATAN: 'ATAN'; @@ -251,6 +252,7 @@ SIN: 'SIN'; SINH: 'SINH'; SQRT: 'SQRT'; SUBDATE: 'SUBDATE'; +SUBTIME: 'SUBTIME'; SUBTRACT: 'SUBTRACT'; SYSDATE: 'SYSDATE'; TAN: 'TAN'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index cc8f5e0a8d..cb2c690316 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -426,6 +426,7 @@ trigonometricFunctionName dateTimeFunctionName : datetimeConstantLiteral | ADDDATE + | ADDTIME | CONVERT_TZ | CURDATE | CURTIME @@ -457,6 +458,7 @@ dateTimeFunctionName | SECOND | SECOND_OF_MINUTE | SUBDATE + | SUBTIME | SYSDATE | TIME | TIME_TO_SEC From 7fa65e786dc78d110e4637511c924a0301c2e827 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 11:16:49 -0800 Subject: [PATCH 07/12] Add Day_Of_Week Function As An Alias Of DayOfWeek (#190) (#1228) (#1239) Added Implementation And Testing For Day_Of_Week Function Signed-off-by: GabeFernandez310 Signed-off-by: GabeFernandez310 (cherry picked from commit bac9c37f3020ea04b581c12c7a492d17ffe9636b) Co-authored-by: GabeFernandez310 --- .../org/opensearch/sql/expression/DSL.java | 10 +- .../expression/datetime/DateTimeFunction.java | 24 ++- .../function/BuiltinFunctionName.java | 1 + .../datetime/DateTimeFunctionTest.java | 162 +++++++++++++++--- docs/user/dql/functions.rst | 15 +- .../sql/sql/DateTimeFunctionIT.java | 43 +++++ sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 + .../sql/sql/antlr/SQLSyntaxParserTest.java | 6 + 8 files changed, 226 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 7899e47a71..00b6b4699c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -326,8 +326,9 @@ public static FunctionExpression dayofmonth(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.DAYOFMONTH, expressions); } - public static FunctionExpression dayofweek(Expression... expressions) { - return compile(FunctionProperties.None, BuiltinFunctionName.DAYOFWEEK, expressions); + public static FunctionExpression dayofweek( + FunctionProperties functionProperties, Expression... expressions) { + return compile(functionProperties, BuiltinFunctionName.DAYOFWEEK, expressions); } public static FunctionExpression dayofyear(Expression... expressions) { @@ -338,6 +339,11 @@ public static FunctionExpression day_of_year(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.DAY_OF_YEAR, expressions); } + public static FunctionExpression day_of_week( + FunctionProperties functionProperties, Expression... expressions) { + return compile(functionProperties, BuiltinFunctionName.DAY_OF_WEEK, expressions); + } + public static FunctionExpression from_days(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.FROM_DAYS, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index ade6170102..4ce70531be 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -111,7 +111,8 @@ public void register(BuiltinFunctionRepository repository) { repository.register(day()); repository.register(dayName()); repository.register(dayOfMonth()); - repository.register(dayOfWeek()); + repository.register(dayOfWeek(BuiltinFunctionName.DAYOFWEEK.getName())); + repository.register(dayOfWeek(BuiltinFunctionName.DAY_OF_WEEK.getName())); repository.register(dayOfYear(BuiltinFunctionName.DAYOFYEAR)); repository.register(dayOfYear(BuiltinFunctionName.DAY_OF_YEAR)); repository.register(from_days()); @@ -449,11 +450,14 @@ private DefaultFunctionResolver dayOfMonth() { } /** - * DAYOFWEEK(STRING/DATE/DATETIME/TIMESTAMP). + * DAYOFWEEK(STRING/DATE/DATETIME/TIME/TIMESTAMP). * return the weekday index for date (1 = Sunday, 2 = Monday, …, 7 = Saturday). */ - private DefaultFunctionResolver dayOfWeek() { - return define(BuiltinFunctionName.DAYOFWEEK.getName(), + private DefaultFunctionResolver dayOfWeek(FunctionName name) { + return define(name, + implWithProperties(nullMissingHandlingWithProperties( + (functionProperties, arg) -> DateTimeFunction.dayOfWeekToday( + functionProperties.getQueryStartClock())), INTEGER, TIME), impl(nullMissingHandling(DateTimeFunction::exprDayOfWeek), INTEGER, DATE), impl(nullMissingHandling(DateTimeFunction::exprDayOfWeek), INTEGER, DATETIME), impl(nullMissingHandling(DateTimeFunction::exprDayOfWeek), INTEGER, TIMESTAMP), @@ -821,6 +825,16 @@ private DefaultFunctionResolver date_format() { ); } + /** + * Day of Week implementation for ExprValue when passing in an arguemt of type TIME. + * + * @param clock Current clock taken from function properties + * @return ExprValue. + */ + private ExprValue dayOfWeekToday(Clock clock) { + return new ExprIntegerValue((formatNow(clock).getDayOfWeek().getValue() % 7) + 1); + } + /** * ADDDATE function implementation for ExprValue. * @@ -1026,7 +1040,7 @@ private ExprValue exprDayOfMonth(ExprValue date) { /** * Day of Week implementation for ExprValue. * - * @param date ExprValue of Date/String type. + * @param date ExprValue of Date/Datetime/String/Timstamp type. * @return ExprValue. */ private ExprValue exprDayOfWeek(ExprValue date) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index cdc0b4e30d..68727ca996 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -71,6 +71,7 @@ public enum BuiltinFunctionName { DAYOFMONTH(FunctionName.of("dayofmonth")), DAYOFWEEK(FunctionName.of("dayofweek")), DAYOFYEAR(FunctionName.of("dayofyear")), + DAY_OF_WEEK(FunctionName.of("day_of_week")), DAY_OF_YEAR(FunctionName.of("day_of_year")), FROM_DAYS(FunctionName.of("from_days")), FROM_UNIXTIME(FunctionName.of("from_unixtime")), diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 7a486fc8d4..980108f41a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -435,32 +435,150 @@ public void dayOfMonth() { assertEquals(integerValue(8), eval(expression)); } + private void dayOfWeekQuery( + FunctionExpression dateExpression, + int dayOfWeek, + String testExpr) { + assertEquals(INTEGER, dateExpression.type()); + assertEquals(integerValue(dayOfWeek), eval(dateExpression)); + assertEquals(testExpr, dateExpression.toString()); + } + @Test public void dayOfWeek() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + FunctionExpression expression1 = DSL.dayofweek( + functionProperties, + DSL.literal(new ExprDateValue("2020-08-07"))); + FunctionExpression expression2 = DSL.dayofweek( + functionProperties, + DSL.literal(new ExprDateValue("2020-08-09"))); + FunctionExpression expression3 = DSL.dayofweek( + functionProperties, + DSL.literal("2020-08-09")); + FunctionExpression expression4 = DSL.dayofweek( + functionProperties, + DSL.literal("2020-08-09 01:02:03")); + + assertAll( + () -> dayOfWeekQuery(expression1, 6, "dayofweek(DATE '2020-08-07')"), + + () -> dayOfWeekQuery(expression2, 1, "dayofweek(DATE '2020-08-09')"), + + () -> dayOfWeekQuery(expression3, 1, "dayofweek(\"2020-08-09\")"), + + () -> dayOfWeekQuery(expression4, 1, "dayofweek(\"2020-08-09 01:02:03\")") + ); + } + + private void dayOfWeekWithUnderscoresQuery( + FunctionExpression dateExpression, + int dayOfWeek, + String testExpr) { + assertEquals(INTEGER, dateExpression.type()); + assertEquals(integerValue(dayOfWeek), eval(dateExpression)); + assertEquals(testExpr, dateExpression.toString()); + } + + @Test + public void dayOfWeekWithUnderscores() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + FunctionExpression expression1 = DSL.day_of_week( + functionProperties, + DSL.literal(new ExprDateValue("2020-08-07"))); + FunctionExpression expression2 = DSL.day_of_week( + functionProperties, + DSL.literal(new ExprDateValue("2020-08-09"))); + FunctionExpression expression3 = DSL.day_of_week( + functionProperties, + DSL.literal("2020-08-09")); + FunctionExpression expression4 = DSL.day_of_week( + functionProperties, + DSL.literal("2020-08-09 01:02:03")); + + assertAll( + () -> dayOfWeekWithUnderscoresQuery(expression1, 6, "day_of_week(DATE '2020-08-07')"), + + () -> dayOfWeekWithUnderscoresQuery(expression2, 1, "day_of_week(DATE '2020-08-09')"), + + () -> dayOfWeekWithUnderscoresQuery(expression3, 1, "day_of_week(\"2020-08-09\")"), + + () -> dayOfWeekWithUnderscoresQuery( + expression4, 1, "day_of_week(\"2020-08-09 01:02:03\")") + ); + } + + @Test + public void testDayOfWeekWithTimeType() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + FunctionExpression expression = DSL.day_of_week( + functionProperties, DSL.literal(new ExprTimeValue("12:23:34"))); + + assertAll( + () -> assertEquals(INTEGER, eval(expression).type()), + () -> assertEquals(( + LocalDate.now( + functionProperties.getQueryStartClock()).getDayOfWeek().getValue() % 7) + 1, + eval(expression).integerValue()), + () -> assertEquals("day_of_week(TIME '12:23:34')", expression.toString()) + ); + } + + private void testInvalidDayOfWeek(String date) { + FunctionExpression expression = DSL.day_of_week( + functionProperties, DSL.literal(new ExprDateValue(date))); + eval(expression); + } + + @Test + public void dayOfWeekWithUnderscoresLeapYear() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + assertAll( + //Feb. 29 of a leap year + () -> dayOfWeekWithUnderscoresQuery(DSL.day_of_week( + functionProperties, + DSL.literal("2020-02-29")), 7, "day_of_week(\"2020-02-29\")"), + //day after Feb. 29 of a leap year + () -> dayOfWeekWithUnderscoresQuery(DSL.day_of_week( + functionProperties, + DSL.literal("2020-03-01")), 1, "day_of_week(\"2020-03-01\")"), + //Feb. 28 of a non-leap year + () -> dayOfWeekWithUnderscoresQuery(DSL.day_of_week( + functionProperties, + DSL.literal("2021-02-28")), 1, "day_of_week(\"2021-02-28\")"), + //Feb. 29 of a non-leap year + () -> assertThrows( + SemanticCheckException.class, () -> testInvalidDayOfWeek("2021-02-29")) + ); + } + + @Test + public void dayOfWeekWithUnderscoresInvalidArgument() { when(nullRef.type()).thenReturn(DATE); when(missingRef.type()).thenReturn(DATE); - assertEquals(nullValue(), eval(DSL.dayofweek(nullRef))); - assertEquals(missingValue(), eval(DSL.dayofweek(missingRef))); - - FunctionExpression expression = DSL.dayofweek(DSL.literal(new ExprDateValue("2020-08-07"))); - assertEquals(INTEGER, expression.type()); - assertEquals("dayofweek(DATE '2020-08-07')", expression.toString()); - assertEquals(integerValue(6), eval(expression)); + assertEquals(nullValue(), eval(DSL.day_of_week(functionProperties, nullRef))); + assertEquals(missingValue(), eval(DSL.day_of_week(functionProperties, missingRef))); - expression = DSL.dayofweek(DSL.literal(new ExprDateValue("2020-08-09"))); - assertEquals(INTEGER, expression.type()); - assertEquals("dayofweek(DATE '2020-08-09')", expression.toString()); - assertEquals(integerValue(1), eval(expression)); + assertAll( + //40th day of the month + () -> assertThrows(SemanticCheckException.class, + () -> testInvalidDayOfWeek("2021-02-40")), - expression = DSL.dayofweek(DSL.literal("2020-08-09")); - assertEquals(INTEGER, expression.type()); - assertEquals("dayofweek(\"2020-08-09\")", expression.toString()); - assertEquals(integerValue(1), eval(expression)); + //13th month of the year + () -> assertThrows(SemanticCheckException.class, + () -> testInvalidDayOfWeek("2021-13-29")), - expression = DSL.dayofweek(DSL.literal("2020-08-09 01:02:03")); - assertEquals(INTEGER, expression.type()); - assertEquals("dayofweek(\"2020-08-09 01:02:03\")", expression.toString()); - assertEquals(integerValue(1), eval(expression)); + //incorrect format + () -> assertThrows(SemanticCheckException.class, + () -> testInvalidDayOfWeek("asdfasdf")) + ); } @Test @@ -486,7 +604,7 @@ public void dayOfYear() { assertEquals(integerValue(220), eval(expression)); } - public void testDayOfYearWithUnderscores(String date, int dayOfYear) { + private void testDayOfYearWithUnderscores(String date, int dayOfYear) { FunctionExpression expression = DSL.day_of_year(DSL.literal(new ExprDateValue(date))); assertEquals(INTEGER, expression.type()); assertEquals(integerValue(dayOfYear), eval(expression)); @@ -553,7 +671,7 @@ public void dayOfYearWithUnderscoresLeapYear() { ); } - public void testInvalidDayOfYear(String date) { + private void testInvalidDayOfYear(String date) { FunctionExpression expression = DSL.day_of_year(DSL.literal(new ExprDateValue(date))); eval(expression); } @@ -792,7 +910,7 @@ public void month() { assertEquals(integerValue(8), eval(expression)); } - public void testInvalidDates(String date) throws SemanticCheckException { + private void testInvalidDates(String date) throws SemanticCheckException { FunctionExpression expression = DSL.month_of_year(DSL.literal(new ExprDateValue(date))); eval(expression); } diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 86d18f1c2f..075f0fb4db 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1515,20 +1515,21 @@ Description Usage: dayofweek(date) returns the weekday index for date (1 = Sunday, 2 = Monday, …, 7 = Saturday). +The `day_of_week` function is also provided as an alias. + Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER Example:: - os> SELECT DAYOFWEEK(DATE('2020-08-26')) + os> SELECT DAYOFWEEK('2020-08-26'), DAY_OF_WEEK('2020-08-26') fetched rows / total rows = 1/1 - +---------------------------------+ - | DAYOFWEEK(DATE('2020-08-26')) | - |---------------------------------| - | 4 | - +---------------------------------+ - + +---------------------------+-----------------------------+ + | DAYOFWEEK('2020-08-26') | DAY_OF_WEEK('2020-08-26') | + |---------------------------+-----------------------------| + | 4 | 4 | + +---------------------------+-----------------------------+ DAYOFYEAR diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 389c976f5c..7015701a34 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -219,6 +219,49 @@ public void testDayOfWeek() throws IOException { verifyDataRows(result, rows(4)); } + @Test + public void testDayOfWeekWithUnderscores() throws IOException { + JSONObject result = executeQuery("select day_of_week(date('2020-09-16'))"); + verifySchema(result, schema("day_of_week(date('2020-09-16'))", null, "integer")); + verifyDataRows(result, rows(4)); + + result = executeQuery("select day_of_week('2020-09-16')"); + verifySchema(result, schema("day_of_week('2020-09-16')", null, "integer")); + verifyDataRows(result, rows(4)); + } + + @Test + public void testDayOfWeekAliasesReturnTheSameResults() throws IOException { + JSONObject result1 = executeQuery("SELECT dayofweek(date('2022-11-22'))"); + JSONObject result2 = executeQuery("SELECT day_of_week(date('2022-11-22'))"); + verifyDataRows(result1, rows(3)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT dayofweek(CAST(date0 AS date)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT day_of_week(CAST(date0 AS date)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT dayofweek(datetime(CAST(time0 AS STRING))) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT day_of_week(datetime(CAST(time0 AS STRING))) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT dayofweek(CAST(time0 AS STRING)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT day_of_week(CAST(time0 AS STRING)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT dayofweek(CAST(datetime0 AS timestamp)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT day_of_week(CAST(datetime0 AS timestamp)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + } + @Test public void testDayOfYear() throws IOException { JSONObject result = executeQuery("select dayofyear(date('2020-09-16'))"); diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index cb2c690316..340409e93a 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -441,6 +441,7 @@ dateTimeFunctionName | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR + | DAY_OF_WEEK | FROM_DAYS | FROM_UNIXTIME | HOUR diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index 487f300dd7..3885e0800f 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -200,6 +200,12 @@ public void can_parse_week_of_year_functions() { assertNotNull(parser.parse("SELECT week_of_year('2022-11-18')")); } + @Test + public void can_parse_day_of_week_functions() { + assertNotNull(parser.parse("SELECT dayofweek('2022-11-18')")); + assertNotNull(parser.parse("SELECT day_of_week('2022-11-18')")); + } + @Test public void can_parse_dayofyear_functions() { assertNotNull(parser.parse("SELECT dayofyear('2022-11-18')")); From 9b2ad5a5064ec75539d6e23b391b976db874cdee Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Mon, 9 Jan 2023 13:13:28 -0800 Subject: [PATCH 08/12] [Backport 2.x] Add Minute_Of_Hour Function As An Alias Of Minute Function (#1253) * Add Minute_Of_Hour Function As An Alias Of Minute Function (#196) (#1230) Added Testing And Implementation For Minute_Of_Hour Function Signed-off-by: GabeFernandez310 Signed-off-by: GabeFernandez310 (cherry picked from commit 61e2374eace01945bb8b083c8b19859e5042e228) * Added Missing Imports Signed-off-by: GabeFernandez310 Signed-off-by: GabeFernandez310 --- .../org/opensearch/sql/expression/DSL.java | 4 + .../expression/datetime/DateTimeFunction.java | 11 ++- .../function/BuiltinFunctionName.java | 1 + .../datetime/DateTimeFunctionTest.java | 81 ++++++++++++++++++- docs/user/dql/functions.rst | 13 +-- .../sql/sql/DateTimeFunctionIT.java | 46 +++++++++++ sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 + .../sql/sql/antlr/SQLSyntaxParserTest.java | 9 +++ 8 files changed, 155 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 00b6b4699c..bb343cd5f9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -364,6 +364,10 @@ public static FunctionExpression minute_of_day(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.MINUTE_OF_DAY, expressions); } + public static FunctionExpression minute_of_hour(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.MINUTE_OF_HOUR, expressions); + } + public static FunctionExpression month(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.MONTH, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 4ce70531be..064d6ea857 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -123,8 +123,9 @@ public void register(BuiltinFunctionRepository repository) { repository.register(makedate()); repository.register(maketime()); repository.register(microsecond()); - repository.register(minute()); + repository.register(minute(BuiltinFunctionName.MINUTE)); repository.register(minute_of_day()); + repository.register(minute(BuiltinFunctionName.MINUTE_OF_HOUR)); repository.register(month(BuiltinFunctionName.MONTH)); repository.register(month(BuiltinFunctionName.MONTH_OF_YEAR)); repository.register(monthName()); @@ -530,11 +531,12 @@ private DefaultFunctionResolver microsecond() { /** * MINUTE(STRING/TIME/DATETIME/TIMESTAMP). return the minute value for time. */ - private DefaultFunctionResolver minute() { - return define(BuiltinFunctionName.MINUTE.getName(), + private DefaultFunctionResolver minute(BuiltinFunctionName name) { + return define(name.getName(), impl(nullMissingHandling(DateTimeFunction::exprMinute), INTEGER, STRING), impl(nullMissingHandling(DateTimeFunction::exprMinute), INTEGER, TIME), impl(nullMissingHandling(DateTimeFunction::exprMinute), INTEGER, DATETIME), + impl(nullMissingHandling(DateTimeFunction::exprMinute), INTEGER, DATE), impl(nullMissingHandling(DateTimeFunction::exprMinute), INTEGER, TIMESTAMP) ); } @@ -1168,7 +1170,8 @@ private ExprValue exprMicrosecond(ExprValue time) { * @return ExprValue. */ private ExprValue exprMinute(ExprValue time) { - return new ExprIntegerValue(time.timeValue().getMinute()); + return new ExprIntegerValue( + (MINUTES.between(LocalTime.MIN, time.timeValue()) % 60)); } /** diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 68727ca996..5572f41a9f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -81,6 +81,7 @@ public enum BuiltinFunctionName { MICROSECOND(FunctionName.of("microsecond")), MINUTE(FunctionName.of("minute")), MINUTE_OF_DAY(FunctionName.of("minute_of_day")), + MINUTE_OF_HOUR(FunctionName.of("minute_of_hour")), MONTH(FunctionName.of("month")), MONTH_OF_YEAR(FunctionName.of("month_of_year")), MONTHNAME(FunctionName.of("monthname")), diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 980108f41a..7761c1e94c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -850,7 +850,7 @@ private void testMinuteOfDay(String date, int value) { assertEquals(INTEGER, expression.type()); assertEquals(integerValue(value), eval(expression)); } - + @Test public void minuteOfDay() { when(nullRef.type()).thenReturn(TIME); @@ -887,6 +887,85 @@ public void minuteOfDay() { testMinuteOfDay("2020-08-17 00:00:01", 0); } + private void minuteOfHourQuery(FunctionExpression dateExpression, int minute, String testExpr) { + assertAll( + () -> assertEquals(INTEGER, dateExpression.type()), + () -> assertEquals(integerValue(minute), eval(dateExpression)), + () -> assertEquals(testExpr, dateExpression.toString()) + ); + } + + private static Stream getTestDataForMinuteOfHour() { + return Stream.of( + Arguments.of( + DSL.literal(new ExprTimeValue("01:02:03")), + 2, + "minute_of_hour(TIME '01:02:03')"), + Arguments.of( + DSL.literal("01:02:03"), + 2, + "minute_of_hour(\"01:02:03\")"), + Arguments.of( + DSL.literal(new ExprTimestampValue("2020-08-17 01:02:03")), + 2, + "minute_of_hour(TIMESTAMP '2020-08-17 01:02:03')"), + Arguments.of( + DSL.literal(new ExprDatetimeValue("2020-08-17 01:02:03")), + 2, + "minute_of_hour(DATETIME '2020-08-17 01:02:03')"), + Arguments.of( + DSL.literal("2020-08-17 01:02:03"), + 2, + "minute_of_hour(\"2020-08-17 01:02:03\")") + ); + } + + @ParameterizedTest(name = "{2}") + @MethodSource("getTestDataForMinuteOfHour") + public void minuteOfHour(LiteralExpression arg, int expectedResult, String expectedString) { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + minuteOfHourQuery(DSL.minute_of_hour(arg), expectedResult, expectedString); + } + + private void invalidMinuteOfHourQuery(String time) { + FunctionExpression expression = DSL.minute_of_hour(DSL.literal(new ExprTimeValue(time))); + eval(expression); + } + + @Test + public void minuteOfHourInvalidArguments() { + when(nullRef.type()).thenReturn(TIME); + when(missingRef.type()).thenReturn(TIME); + + assertAll( + () -> assertEquals(nullValue(), eval(DSL.minute_of_hour(nullRef))), + () -> assertEquals(missingValue(), eval(DSL.minute_of_hour(missingRef))), + + //Invalid Seconds + () -> assertThrows( + SemanticCheckException.class, + () -> invalidMinuteOfHourQuery("12:23:61")), + + //Invalid Minutes + () -> assertThrows( + SemanticCheckException.class, + () -> invalidMinuteOfHourQuery("12:61:34")), + + //Invalid Hours + () -> assertThrows( + SemanticCheckException.class, + () -> invalidMinuteOfHourQuery("25:23:34")), + + //incorrect format + () -> assertThrows( + SemanticCheckException.class, + () -> invalidMinuteOfHourQuery("asdfasdf")) + ); + } + + @Test public void month() { when(nullRef.type()).thenReturn(DATE); diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 075f0fb4db..9a824fe449 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1829,6 +1829,7 @@ Description >>>>>>>>>>> Usage: minute(time) returns the minute for time, in the range 0 to 59. +The `minute_of_hour` function is provided as an alias. Argument type: STRING/TIME/DATETIME/TIMESTAMP @@ -1836,13 +1837,13 @@ Return type: INTEGER Example:: - os> SELECT MINUTE((TIME '01:02:03')) + os> SELECT MINUTE(time('01:02:03')), MINUTE_OF_HOUR(time('01:02:03')) fetched rows / total rows = 1/1 - +-----------------------------+ - | MINUTE((TIME '01:02:03')) | - |-----------------------------| - | 2 | - +-----------------------------+ + +----------------------------+------------------------------------+ + | MINUTE(time('01:02:03')) | MINUTE_OF_HOUR(time('01:02:03')) | + |----------------------------+------------------------------------| + | 2 | 2 | + +----------------------------+------------------------------------+ MINUTE_OF_DAY ------ diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 7015701a34..85661bfa97 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -433,6 +433,52 @@ public void testMinuteOfDay() throws IOException { verifyDataRows(result, rows(1050)); } + @Test + public void testMinuteOfHour() throws IOException { + JSONObject result = executeQuery("select minute_of_hour(timestamp('2020-09-16 17:30:00'))"); + verifySchema(result, schema( + "minute_of_hour(timestamp('2020-09-16 17:30:00'))", null, "integer")); + verifyDataRows(result, rows(30)); + + result = executeQuery("select minute_of_hour(time('17:30:00'))"); + verifySchema(result, schema("minute_of_hour(time('17:30:00'))", null, "integer")); + verifyDataRows(result, rows(30)); + + result = executeQuery("select minute_of_hour('2020-09-16 17:30:00')"); + verifySchema(result, schema("minute_of_hour('2020-09-16 17:30:00')", null, "integer")); + verifyDataRows(result, rows(30)); + + result = executeQuery("select minute_of_hour('17:30:00')"); + verifySchema(result, schema("minute_of_hour('17:30:00')", null, "integer")); + verifyDataRows(result, rows(30)); + } + + @Test + public void testMinuteFunctionAliasesReturnTheSameResults() throws IOException { + JSONObject result1 = executeQuery("SELECT minute('11:30:00')"); + JSONObject result2 = executeQuery("SELECT minute_of_hour('11:30:00')"); + verifyDataRows(result1, rows(30)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT minute(datetime(CAST(time0 AS STRING))) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT minute_of_hour(datetime(CAST(time0 AS STRING))) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT minute(CAST(time0 AS STRING)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT minute_of_hour(CAST(time0 AS STRING)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT minute(CAST(datetime0 AS timestamp)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT minute_of_hour(CAST(datetime0 AS timestamp)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + } + @Test public void testMonth() throws IOException { JSONObject result = executeQuery("select month(date('2020-09-16'))"); diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 340409e93a..5f468690d5 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -450,6 +450,7 @@ dateTimeFunctionName | MICROSECOND | MINUTE | MINUTE_OF_DAY + | MINUTE_OF_HOUR | MONTH | MONTHNAME | NOW diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index 3885e0800f..5716f17098 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -212,6 +212,15 @@ public void can_parse_dayofyear_functions() { assertNotNull(parser.parse("SELECT day_of_year('2022-11-18')")); } + @Test + public void can_parse_minute_functions() { + assertNotNull(parser.parse("SELECT minute('12:23:34')")); + assertNotNull(parser.parse("SELECT minute_of_hour('12:23:34')")); + + assertNotNull(parser.parse("SELECT minute('2022-12-20 12:23:34')")); + assertNotNull(parser.parse("SELECT minute_of_hour('2022-12-20 12:23:34')")); + } + @Test public void can_parse_month_of_year_function() { assertNotNull(parser.parse("SELECT month('2022-11-18')")); From 4e13ae75ce0d38b220d1e02b9e69eca86ec53ea3 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:16:11 -0800 Subject: [PATCH 09/12] Add support for long value return for CEIL, CEILING and FLOOR math functions (#1205) (#1255) * Added long fix for CEIL, CEILING and FLOOR functions using LONG instead of INT for RETURN. Signed-off-by: MitchellGale-BitQuill Signed-off-by: Yury-Fridlyand Co-authored-by: Yury-Fridlyand --- .../arthmetic/MathematicalFunction.java | 12 ++-- .../arthmetic/MathematicalFunctionTest.java | 69 ++++++++++++------- docs/user/dql/functions.rst | 67 ++++++++++++++++-- docs/user/ppl/functions/math.rst | 69 ++++++++++++++----- .../sql/legacy/TypeInformationIT.java | 2 +- .../sql/ppl/MathematicalFunctionIT.java | 6 +- .../sql/sql/MathematicalFunctionIT.java | 15 ++++ .../expressions/mathematical_functions.txt | 22 +++--- 8 files changed, 195 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java b/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java index 4424243860..20b7928307 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java @@ -119,16 +119,16 @@ private static DefaultFunctionResolver abs() { private static DefaultFunctionResolver ceil() { return FunctionDSL.define(BuiltinFunctionName.CEIL.getName(), FunctionDSL.impl( - FunctionDSL.nullMissingHandling(v -> new ExprIntegerValue(Math.ceil(v.doubleValue()))), - INTEGER, DOUBLE) + FunctionDSL.nullMissingHandling(v -> new ExprLongValue(Math.ceil(v.doubleValue()))), + LONG, DOUBLE) ); } private static DefaultFunctionResolver ceiling() { return FunctionDSL.define(BuiltinFunctionName.CEILING.getName(), FunctionDSL.impl( - FunctionDSL.nullMissingHandling(v -> new ExprIntegerValue(Math.ceil(v.doubleValue()))), - INTEGER, DOUBLE) + FunctionDSL.nullMissingHandling(v -> new ExprLongValue(Math.ceil(v.doubleValue()))), + LONG, DOUBLE) ); } @@ -204,8 +204,8 @@ private static DefaultFunctionResolver exp() { private static DefaultFunctionResolver floor() { return FunctionDSL.define(BuiltinFunctionName.FLOOR.getName(), FunctionDSL.impl( - FunctionDSL.nullMissingHandling(v -> new ExprIntegerValue(Math.floor(v.doubleValue()))), - INTEGER, DOUBLE) + FunctionDSL.nullMissingHandling(v -> new ExprLongValue(Math.floor(v.doubleValue()))), + LONG, DOUBLE) ); } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java index d6d15e9315..3a03ba79ad 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java @@ -191,13 +191,13 @@ public void ceil_int_value(Integer value) { FunctionExpression ceil = DSL.ceil(DSL.literal(value)); assertThat( ceil.valueOf(valueEnv()), - allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceil(%s)", value), ceil.toString()); + allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceil(%s)", value.toString()), ceil.toString()); FunctionExpression ceiling = DSL.ceiling(DSL.literal(value)); assertThat( - ceiling.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceiling(%s)", value), ceiling.toString()); + ceiling.valueOf(valueEnv()), allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceiling(%s)", value.toString()), ceiling.toString()); } /** @@ -208,13 +208,30 @@ public void ceil_int_value(Integer value) { public void ceil_long_value(Long value) { FunctionExpression ceil = DSL.ceil(DSL.literal(value)); assertThat( - ceil.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceil(%s)", value), ceil.toString()); + ceil.valueOf(valueEnv()), allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceil(%s)", value.toString()), ceil.toString()); FunctionExpression ceiling = DSL.ceiling(DSL.literal(value)); assertThat( - ceiling.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceiling(%s)", value), ceiling.toString()); + ceiling.valueOf(valueEnv()), allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceiling(%s)", value.toString()), ceiling.toString()); + } + + /** + * Test ceil/ceiling with long value. + */ + @ParameterizedTest(name = "ceil({0})") + @ValueSource(longs = {9223372036854775805L, -9223372036854775805L}) + public void ceil_long_value_long(Long value) { + FunctionExpression ceil = DSL.ceil(DSL.literal(value)); + assertThat( + ceil.valueOf(valueEnv()), allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceil(%s)", value.toString()), ceil.toString()); + + FunctionExpression ceiling = DSL.ceiling(DSL.literal(value)); + assertThat( + ceiling.valueOf(valueEnv()), allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceiling(%s)", value.toString()), ceiling.toString()); } /** @@ -225,13 +242,13 @@ public void ceil_long_value(Long value) { public void ceil_float_value(Float value) { FunctionExpression ceil = DSL.ceil(DSL.literal(value)); assertThat( - ceil.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceil(%s)", value), ceil.toString()); + ceil.valueOf(valueEnv()), allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceil(%s)", value.toString()), ceil.toString()); FunctionExpression ceiling = DSL.ceiling(DSL.literal(value)); assertThat( - ceiling.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceiling(%s)", value), ceiling.toString()); + ceiling.valueOf(valueEnv()), allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceiling(%s)", value.toString()), ceiling.toString()); } /** @@ -242,13 +259,13 @@ public void ceil_float_value(Float value) { public void ceil_double_value(Double value) { FunctionExpression ceil = DSL.ceil(DSL.literal(value)); assertThat( - ceil.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceil(%s)", value), ceil.toString()); + ceil.valueOf(valueEnv()), allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceil(%s)", value.toString()), ceil.toString()); FunctionExpression ceiling = DSL.ceiling(DSL.literal(value)); assertThat( - ceiling.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceiling(%s)", value), ceiling.toString()); + ceiling.valueOf(valueEnv()), allOf(hasType(LONG), hasValue((long) Math.ceil(value)))); + assertEquals(String.format("ceiling(%s)", value.toString()), ceiling.toString()); } /** @@ -257,11 +274,11 @@ public void ceil_double_value(Double value) { @Test public void ceil_null_value() { FunctionExpression ceil = DSL.ceil(DSL.ref(DOUBLE_TYPE_NULL_VALUE_FIELD, DOUBLE)); - assertEquals(INTEGER, ceil.type()); + assertEquals(LONG, ceil.type()); assertTrue(ceil.valueOf(valueEnv()).isNull()); FunctionExpression ceiling = DSL.ceiling(DSL.ref(DOUBLE_TYPE_NULL_VALUE_FIELD, DOUBLE)); - assertEquals(INTEGER, ceiling.type()); + assertEquals(LONG, ceiling.type()); assertTrue(ceiling.valueOf(valueEnv()).isNull()); } @@ -271,11 +288,11 @@ public void ceil_null_value() { @Test public void ceil_missing_value() { FunctionExpression ceil = DSL.ceil(DSL.ref(DOUBLE_TYPE_MISSING_VALUE_FIELD, DOUBLE)); - assertEquals(INTEGER, ceil.type()); + assertEquals(LONG, ceil.type()); assertTrue(ceil.valueOf(valueEnv()).isMissing()); FunctionExpression ceiling = DSL.ceiling(DSL.ref(DOUBLE_TYPE_MISSING_VALUE_FIELD, DOUBLE)); - assertEquals(INTEGER, ceiling.type()); + assertEquals(LONG, ceiling.type()); assertTrue(ceiling.valueOf(valueEnv()).isMissing()); } @@ -557,7 +574,7 @@ public void floor_int_value(Integer value) { FunctionExpression floor = DSL.floor(DSL.literal(value)); assertThat( floor.valueOf(valueEnv()), - allOf(hasType(INTEGER), hasValue((int) Math.floor(value)))); + allOf(hasType(LONG), hasValue((long) Math.floor(value)))); assertEquals(String.format("floor(%s)", value.toString()), floor.toString()); } @@ -570,7 +587,7 @@ public void floor_long_value(Long value) { FunctionExpression floor = DSL.floor(DSL.literal(value)); assertThat( floor.valueOf(valueEnv()), - allOf(hasType(INTEGER), hasValue((int) Math.floor(value)))); + allOf(hasType(LONG), hasValue((long) Math.floor(value)))); assertEquals(String.format("floor(%s)", value.toString()), floor.toString()); } @@ -583,7 +600,7 @@ public void floor_float_value(Float value) { FunctionExpression floor = DSL.floor(DSL.literal(value)); assertThat( floor.valueOf(valueEnv()), - allOf(hasType(INTEGER), hasValue((int) Math.floor(value)))); + allOf(hasType(LONG), hasValue((long) Math.floor(value)))); assertEquals(String.format("floor(%s)", value.toString()), floor.toString()); } @@ -596,7 +613,7 @@ public void floor_double_value(Double value) { FunctionExpression floor = DSL.floor(DSL.literal(value)); assertThat( floor.valueOf(valueEnv()), - allOf(hasType(INTEGER), hasValue((int) Math.floor(value)))); + allOf(hasType(LONG), hasValue((long) Math.floor(value)))); assertEquals(String.format("floor(%s)", value.toString()), floor.toString()); } @@ -606,7 +623,7 @@ public void floor_double_value(Double value) { @Test public void floor_null_value() { FunctionExpression floor = DSL.floor(DSL.ref(DOUBLE_TYPE_NULL_VALUE_FIELD, DOUBLE)); - assertEquals(INTEGER, floor.type()); + assertEquals(LONG, floor.type()); assertTrue(floor.valueOf(valueEnv()).isNull()); } @@ -616,7 +633,7 @@ public void floor_null_value() { @Test public void floor_missing_value() { FunctionExpression floor = DSL.floor(DSL.ref(DOUBLE_TYPE_MISSING_VALUE_FIELD, DOUBLE)); - assertEquals(INTEGER, floor.type()); + assertEquals(LONG, floor.type()); assertTrue(floor.valueOf(valueEnv()).isMissing()); } diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 9a824fe449..3dbfedd143 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -231,12 +231,42 @@ Example:: CEIL ---- +An alias for `CEILING`_ function. + + +CEILING +------- + Description >>>>>>>>>>> -Specifications: +Usage: CEILING(T) takes the ceiling of value T. + +Note: `CEIL`_ and CEILING functions have the same implementation & functionality + +Limitation: CEILING only works as expected when IEEE 754 double type displays decimal when stored. + +Argument type: INTEGER/LONG/FLOAT/DOUBLE + +Return type: LONG + +Example:: + + os> SELECT CEILING(0), CEILING(50.00005), CEILING(-50.00005); + fetched rows / total rows = 1/1 + +--------------+---------------------+----------------------+ + | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | + |--------------+---------------------+----------------------| + | 0 | 51 | -50 | + +--------------+---------------------+----------------------+ -1. CEIL(NUMBER T) -> T + os> SELECT CEILING(3147483647.12345), CEILING(113147483647.12345), CEILING(3147483647.00001); + fetched rows / total rows = 1/1 + +-----------------------------+-------------------------------+-----------------------------+ + | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | + |-----------------------------+-------------------------------+-----------------------------| + | 3147483648 | 113147483648 | 3147483648 | + +-----------------------------+-------------------------------+-----------------------------+ CONV @@ -424,10 +454,39 @@ FLOOR Description >>>>>>>>>>> -Specifications: +Usage: FLOOR(T) takes the floor of value T. + +Limitation: FLOOR only works as expected when IEEE 754 double type displays decimal when stored. + +Argument type: INTEGER/LONG/FLOAT/DOUBLE + +Return type: LONG + +Example:: -1. FLOOR(NUMBER T) -> T + os> SELECT FLOOR(0), FLOOR(50.00005), FLOOR(-50.00005); + fetched rows / total rows = 1/1 + +------------+-------------------+--------------------+ + | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | + |------------+-------------------+--------------------| + | 0 | 50 | -51 | + +------------+-------------------+--------------------+ + os> SELECT FLOOR(3147483647.12345), FLOOR(113147483647.12345), FLOOR(3147483647.00001); + fetched rows / total rows = 1/1 + +---------------------------+-----------------------------+---------------------------+ + | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | + |---------------------------+-----------------------------+---------------------------| + | 3147483647 | 113147483647 | 3147483647 | + +---------------------------+-----------------------------+---------------------------+ + + os> SELECT FLOOR(282474973688888.022), FLOOR(9223372036854775807.022), FLOOR(9223372036854775807.0000001); + fetched rows / total rows = 1/1 + +------------------------------+----------------------------------+--------------------------------------+ + | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | + |------------------------------+----------------------------------+--------------------------------------| + | 282474973688888 | 9223372036854775807 | 9223372036854775807 | + +------------------------------+----------------------------------+--------------------------------------+ LN -- diff --git a/docs/user/ppl/functions/math.rst b/docs/user/ppl/functions/math.rst index 69f94091e7..20bd1d6a70 100644 --- a/docs/user/ppl/functions/math.rst +++ b/docs/user/ppl/functions/math.rst @@ -127,24 +127,42 @@ Example:: CEIL ---- +An alias for `CEILING`_ function. + + +CEILING +------- + Description >>>>>>>>>>> -Usage: ceil(x) return the smallest integer value this is greater or equal to x. +Usage: CEILING(T) takes the ceiling of value T. + +Note: `CEIL`_ and CEILING functions have the same implementation & functionality + +Limitation: CEILING only works as expected when IEEE 754 double type displays decimal when stored. Argument type: INTEGER/LONG/FLOAT/DOUBLE -Return type: INTEGER +Return type: LONG Example:: - os> source=people | eval `CEIL(2.75)` = CEIL(2.75) | fields `CEIL(2.75)` + os> source=people | eval `CEILING(0)` = CEILING(0), `CEILING(50.00005)` = CEILING(50.00005), `CEILING(-50.00005)` = CEILING(-50.00005) | fields `CEILING(0)`, `CEILING(50.00005)`, `CEILING(-50.00005)` fetched rows / total rows = 1/1 - +--------------+ - | CEIL(2.75) | - |--------------| - | 3 | - +--------------+ + +--------------+---------------------+----------------------+ + | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | + |--------------+---------------------+----------------------| + | 0 | 51 | -50 | + +--------------+---------------------+----------------------+ + + os> source=people | eval `CEILING(3147483647.12345)` = CEILING(3147483647.12345), `CEILING(113147483647.12345)` = CEILING(113147483647.12345), `CEILING(3147483647.00001)` = CEILING(3147483647.00001) | fields `CEILING(3147483647.12345)`, `CEILING(113147483647.12345)`, `CEILING(3147483647.00001)` + fetched rows / total rows = 1/1 + +-----------------------------+-------------------------------+-----------------------------+ + | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | + |-----------------------------+-------------------------------+-----------------------------| + | 3147483648 | 113147483648 | 3147483648 | + +-----------------------------+-------------------------------+-----------------------------+ CONV @@ -310,22 +328,39 @@ FLOOR Description >>>>>>>>>>> -Usage: floor(x) return the largest integer value this is smaller or equal to x. +Usage: FLOOR(T) takes the floor of value T. -Argument type: INTEGER/LONG/FLOAT/DOUBLE +Limitation: FLOOR only works as expected when IEEE 754 double type displays decimal when stored. -Return type: INTEGER +Argument type: a: INTEGER/LONG/FLOAT/DOUBLE + +Return type: LONG Example:: - os> source=people | eval `FLOOR(2.75)` = FLOOR(2.75) | fields `FLOOR(2.75)` + os> source=people | eval `FLOOR(0)` = FLOOR(0), `FLOOR(50.00005)` = FLOOR(50.00005), `FLOOR(-50.00005)` = FLOOR(-50.00005) | fields `FLOOR(0)`, `FLOOR(50.00005)`, `FLOOR(-50.00005)` fetched rows / total rows = 1/1 - +---------------+ - | FLOOR(2.75) | - |---------------| - | 2 | - +---------------+ + +------------+-------------------+--------------------+ + | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | + |------------+-------------------+--------------------| + | 0 | 50 | -51 | + +------------+-------------------+--------------------+ + os> source=people | eval `FLOOR(3147483647.12345)` = FLOOR(3147483647.12345), `FLOOR(113147483647.12345)` = FLOOR(113147483647.12345), `FLOOR(3147483647.00001)` = FLOOR(3147483647.00001) | fields `FLOOR(3147483647.12345)`, `FLOOR(113147483647.12345)`, `FLOOR(3147483647.00001)` + fetched rows / total rows = 1/1 + +---------------------------+-----------------------------+---------------------------+ + | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | + |---------------------------+-----------------------------+---------------------------| + | 3147483647 | 113147483647 | 3147483647 | + +---------------------------+-----------------------------+---------------------------+ + + os> source=people | eval `FLOOR(282474973688888.022)` = FLOOR(282474973688888.022), `FLOOR(9223372036854775807.022)` = FLOOR(9223372036854775807.022), `FLOOR(9223372036854775807.0000001)` = FLOOR(9223372036854775807.0000001) | fields `FLOOR(282474973688888.022)`, `FLOOR(9223372036854775807.022)`, `FLOOR(9223372036854775807.0000001)` + fetched rows / total rows = 1/1 + +------------------------------+----------------------------------+--------------------------------------+ + | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | + |------------------------------+----------------------------------+--------------------------------------| + | 282474973688888 | 9223372036854775807 | 9223372036854775807 | + +------------------------------+----------------------------------+--------------------------------------+ LN -- diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java index 01c8d33467..2bd3835a3a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java @@ -40,7 +40,7 @@ public void testCeilWithLongFieldReturnsLong() { executeJdbcRequest("SELECT CEIL(balance) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY balance LIMIT 5"); - verifySchema(response, schema("CEIL(balance)", null, "integer")); + verifySchema(response, schema("CEIL(balance)", null, "long")); } /* diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java index f6fe93dc5f..6dd2d3916f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java @@ -46,7 +46,7 @@ public void testCeil() throws IOException { executeQuery( String.format( "source=%s | eval f = ceil(age) | fields f", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "integer")); + verifySchema(result, schema("f", null, "long")); verifyDataRows( result, rows(32), rows(36), rows(28), rows(33), rows(36), rows(39), rows(34)); @@ -58,7 +58,7 @@ public void testCeiling() throws IOException { executeQuery( String.format( "source=%s | eval f = ceiling(age) | fields f", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "integer")); + verifySchema(result, schema("f", null, "long")); verifyDataRows( result, rows(32), rows(36), rows(28), rows(33), rows(36), rows(39), rows(34)); @@ -94,7 +94,7 @@ public void testFloor() throws IOException { executeQuery( String.format( "source=%s | eval f = floor(age) | fields f", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "integer")); + verifySchema(result, schema("f", null, "long")); verifyDataRows( result, rows(32), rows(36), rows(28), rows(33), rows(36), rows(39), rows(34)); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java index f2d1bb7d28..b8767eb2f1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java @@ -40,6 +40,21 @@ public void testPI() throws IOException { verifyDataRows(result, rows(3.141592653589793)); } + @Test + public void testCeil() throws IOException { + JSONObject result = executeQuery("select ceil(0)"); + verifySchema(result, schema("ceil(0)", null, "long")); + verifyDataRows(result, rows(0)); + + result = executeQuery("select ceil(2147483646.9)"); + verifySchema(result, schema("ceil(2147483646.9)", null, "long")); + verifyDataRows(result, rows(2147483647)); + + result = executeQuery("select ceil(92233720368547807.9)"); + verifySchema(result, schema("ceil(92233720368547807.9)", null, "long")); + verifyDataRows(result, rows(92233720368547808L)); + } + @Test public void testConv() throws IOException { JSONObject result = executeQuery("select conv(11, 10, 16)"); diff --git a/integ-test/src/test/resources/correctness/expressions/mathematical_functions.txt b/integ-test/src/test/resources/correctness/expressions/mathematical_functions.txt index c99ca6c7a9..cb479e2eba 100644 --- a/integ-test/src/test/resources/correctness/expressions/mathematical_functions.txt +++ b/integ-test/src/test/resources/correctness/expressions/mathematical_functions.txt @@ -5,20 +5,22 @@ abs(-1.234) abs(0.0) abs(4.321) abs(abs(-1.2) * -1) -ceil(1) -ceil(-1) -ceil(0.0) -ceil(0.4999) -ceil(abs(1)) +# During comparison with H2 and SQLite it expects ceil and floor to be stored as INT values. This casts to resolve. +CAST(ceil(1) AS INT) +CAST(ceil(-1) AS INT) +CAST(ceil(0.0) AS INT) +CAST(ceil(0.4999) AS INT) +CAST(ceil(abs(1)) AS INT) +# CAST(CEIL(2147483647 + 0.6) AS INT) will fail because the cast limits the return to be INT_MAX (21474883647) which is not an H2 or SQLite limitation exp(0) exp(1) exp(-1) exp(exp(1) + ceil(-1)) -floor(1) -floor(-1) -floor(0.0) -floor(0.4999) -floor(abs(-1)) +CAST(floor(1) AS INT) +CAST(floor(-1) AS INT) +CAST(floor(0.0) AS INT) +CAST(floor(0.4999) AS INT) +CAST(floor(abs(-1)) AS INT) log(2) log(2.1) log(log(2)) From a00f32158340621311284444c80ec2e970b6c6bc Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:37:11 -0800 Subject: [PATCH 10/12] Support JOIN query on object field with unexpanded name (#1229) (#1250) * Resolve sub object field in search hit source Signed-off-by: Chen Dai * Rename to unexpanded object Signed-off-by: Chen Dai * Update IT with where condition Signed-off-by: Chen Dai * Fix test index mapping Signed-off-by: Chen Dai Signed-off-by: Chen Dai (cherry picked from commit 151f4cc877c9964b6ce2d7118d3249c5b42eb941) Co-authored-by: Chen Dai --- .../org/opensearch/sql/legacy/HashJoinIT.java | 21 ++++++ .../sql/legacy/SQLIntegTestCase.java | 5 ++ .../org/opensearch/sql/legacy/TestUtils.java | 5 ++ .../opensearch/sql/legacy/TestsConstants.java | 1 + .../unexpanded_object_index_mapping.json | 27 ++++++++ .../test/resources/unexpanded_objects.json | 8 +++ .../physical/node/scroll/SearchHitRow.java | 7 ++ .../node/scroll/SearchHitRowTest.java | 65 +++++++++++++++++++ 8 files changed, 139 insertions(+) create mode 100644 integ-test/src/test/resources/indexDefinitions/unexpanded_object_index_mapping.json create mode 100644 integ-test/src/test/resources/unexpanded_objects.json create mode 100644 legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/HashJoinIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/HashJoinIT.java index 284d034cd8..9cd497e675 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/HashJoinIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/HashJoinIT.java @@ -9,6 +9,11 @@ import static org.hamcrest.Matchers.equalTo; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_GAME_OF_THRONES; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_UNEXPANDED_OBJECT; +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; import java.util.HashSet; @@ -55,6 +60,7 @@ public class HashJoinIT extends SQLIntegTestCase { protected void init() throws Exception { loadIndex(Index.ACCOUNT); loadIndex(Index.GAME_OF_THRONES); + loadIndex(Index.UNEXPANDED_OBJECT); } @Test @@ -69,6 +75,21 @@ public void leftJoin() throws IOException { testJoin("LEFT JOIN"); } + @Test + public void innerJoinUnexpandedObjectField() { + String query = String.format(Locale.ROOT, + "SELECT " + + "a.id.serial, b.id.serial " + + "FROM %1$s AS a " + + "JOIN %1$s AS b " + + "ON a.id.serial = b.attributes.hardware.correlate_id " + + "WHERE b.attributes.hardware.platform = 'Linux' ", + TEST_INDEX_UNEXPANDED_OBJECT); + + JSONObject response = executeJdbcRequest(query); + verifyDataRows(response, rows(3, 1), rows(3, 3)); + } + @Test public void innerJoinWithObjectField() throws IOException { testJoinWithObjectField("INNER JOIN", ""); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 80348b2a8b..0cfc4a6aa6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -57,6 +57,7 @@ import static org.opensearch.sql.legacy.TestUtils.getPhraseIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import static org.opensearch.sql.legacy.TestUtils.getStringIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getUnexpandedObjectIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getWeblogsIndexMapping; import static org.opensearch.sql.legacy.TestUtils.isIndexExist; import static org.opensearch.sql.legacy.TestUtils.loadDataByRestClient; @@ -517,6 +518,10 @@ public enum Index { "joinType", getJoinTypeIndexMapping(), "src/test/resources/join_objects.json"), + UNEXPANDED_OBJECT(TestsConstants.TEST_INDEX_UNEXPANDED_OBJECT, + "unexpandedObject", + getUnexpandedObjectIndexMapping(), + "src/test/resources/unexpanded_objects.json"), BANK(TestsConstants.TEST_INDEX_BANK, "account", getBankIndexMapping(), diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 8f8ee4a70f..30cee86e15 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -188,6 +188,11 @@ public static String getJoinTypeIndexMapping() { return getMappingFile(mappingFile); } + public static String getUnexpandedObjectIndexMapping() { + String mappingFile = "unexpanded_object_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getBankIndexMapping() { String mappingFile = "bank_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index aff269fcce..c79314af6a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -36,6 +36,7 @@ public class TestsConstants { TEST_INDEX + "_nested_type_with_quotes"; public final static String TEST_INDEX_EMPLOYEE_NESTED = TEST_INDEX + "_employee_nested"; public final static String TEST_INDEX_JOIN_TYPE = TEST_INDEX + "_join_type"; + public final static String TEST_INDEX_UNEXPANDED_OBJECT = TEST_INDEX + "_unexpanded_object"; public final static String TEST_INDEX_BANK = TEST_INDEX + "_bank"; public final static String TEST_INDEX_BANK_TWO = TEST_INDEX_BANK + "_two"; public final static String TEST_INDEX_BANK_WITH_NULL_VALUES = diff --git a/integ-test/src/test/resources/indexDefinitions/unexpanded_object_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/unexpanded_object_index_mapping.json new file mode 100644 index 0000000000..8275147375 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/unexpanded_object_index_mapping.json @@ -0,0 +1,27 @@ +{ + "mappings": { + "properties": { + "id": { + "properties": { + "serial": { + "type": "integer" + } + } + }, + "attributes": { + "properties": { + "hardware": { + "properties": { + "correlate_id": { + "type": "integer" + }, + "platform": { + "type": "keyword" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/unexpanded_objects.json b/integ-test/src/test/resources/unexpanded_objects.json new file mode 100644 index 0000000000..e1df2570cc --- /dev/null +++ b/integ-test/src/test/resources/unexpanded_objects.json @@ -0,0 +1,8 @@ +{"index":{"_id":"1"}} +{"id.serial" : 1 , "attributes.hardware.correlate_id": 3, "attributes.hardware.platform": "Linux"} +{"index":{"_id":"2"}} +{"id.serial" : 2 , "attributes.hardware.correlate_id": 3, "attributes.hardware.platform": "Windows"} +{"index":{"_id":"3"}} +{"id.serial" : 3 , "attributes.hardware.correlate_id": 3, "attributes.hardware.platform": "Linux"} +{"index":{"_id":"4"}} +{"id.serial" : 4 , "attributes.hardware.correlate_id": 100, "attributes.hardware.platform": "Linux"} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java index 8f418deadb..b8cc2bb965 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java @@ -134,6 +134,13 @@ private Object getValueOfPath(Object source, String path, boolean isIgnoreFirstD if (dot == -1) { return ((Map) source).get(path); } + + // Object field name maybe unexpanded without recursive object structure + // ex. {"a.b.c": value} instead of {"a": {"b": {"c": value}}}} + if (((Map) source).containsKey(path)) { + return ((Map) source).get(path); + } + return getValueOfPath( ((Map) source).get(path.substring(0, dot)), path.substring(dot + 1), diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java new file mode 100644 index 0000000000..6c2f789a47 --- /dev/null +++ b/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy.query.planner.physical.node.scroll; + +import static org.junit.Assert.assertEquals; +import static org.opensearch.sql.legacy.query.planner.physical.Row.RowKey; + +import com.google.common.collect.ImmutableMap; +import org.junit.Test; +import org.opensearch.common.bytes.BytesArray; +import org.opensearch.search.SearchHit; + +public class SearchHitRowTest { + + @Test + public void testKeyWithObjectField() { + SearchHit hit = new SearchHit(1); + hit.sourceRef(new BytesArray("{\"id\": {\"serial\": 3}}")); + SearchHitRow row = new SearchHitRow(hit, "a"); + RowKey key = row.key(new String[]{"id.serial"}); + + Object[] data = key.keys(); + assertEquals(1, data.length); + assertEquals(3, data[0]); + } + + @Test + public void testKeyWithUnexpandedObjectField() { + SearchHit hit = new SearchHit(1); + hit.sourceRef(new BytesArray("{\"attributes.hardware.correlate_id\": 10}")); + SearchHitRow row = new SearchHitRow(hit, "a"); + RowKey key = row.key(new String[]{"attributes.hardware.correlate_id"}); + + Object[] data = key.keys(); + assertEquals(1, data.length); + assertEquals(10, data[0]); + } + + @Test + public void testRetainWithObjectField() { + SearchHit hit = new SearchHit(1); + hit.sourceRef(new BytesArray("{\"a.id\": {\"serial\": 3}}")); + SearchHitRow row = new SearchHitRow(hit, ""); + row.retain(ImmutableMap.of("a.id.serial", "")); + + SearchHit expected = new SearchHit(1); + expected.sourceRef(new BytesArray("{\"a.id\": {\"serial\": 3}}")); + assertEquals(expected, row.data()); + } + + @Test + public void testRetainWithUnexpandedObjectField() { + SearchHit hit = new SearchHit(1); + hit.sourceRef(new BytesArray("{\"a.attributes.hardware.correlate_id\": 10}")); + SearchHitRow row = new SearchHitRow(hit, ""); + row.retain(ImmutableMap.of("a.attributes.hardware.correlate_id", "")); + + SearchHit expected = new SearchHit(1); + expected.sourceRef(new BytesArray("{\"a.attributes.hardware.correlate_id\": 10}")); + assertEquals(expected, row.data()); + } +} \ No newline at end of file From fa1083bea6818c57cdc18b1b6b052e41f1c04f99 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 9 Jan 2023 13:55:48 -0800 Subject: [PATCH 11/12] Remove unnecessary scripts after repo split (#1256) Signed-off-by: Joshua Li --- build.gradle | 7 ----- scripts/build.sh | 82 ------------------------------------------------ 2 files changed, 89 deletions(-) delete mode 100755 scripts/build.sh diff --git a/build.gradle b/build.gradle index 846bff7d34..c3b45e8f95 100644 --- a/build.gradle +++ b/build.gradle @@ -190,12 +190,5 @@ task updateVersion { // String tokenization to support -SNAPSHOT ant.replaceregexp(file:'.github/workflows/sql-workbench-test-and-build-workflow.yml', match:'OPENSEARCH_PLUGIN_VERSION: \\d+.\\d+.\\d+.\\d+', replace:'OPENSEARCH_PLUGIN_VERSION: ' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true) ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) - ant.replaceregexp(match:'"version": "\\d+.\\d+.\\d+.\\d+', replace:'"version": ' + '"' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true) { - fileset(dir: projectDir) { - include(name: "workbench/package.json") - include(name: "workbench/opensearch_dashboards.json") - } - } - ant.replaceregexp(file:'workbench/opensearch_dashboards.json', match:'"opensearchDashboardsVersion": "\\d+.\\d+.\\d+', replace:'"opensearchDashboardsVersion": ' + '"' + newVersion.tokenize('-')[0], flags:'g', byline:true) } } diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100755 index 4b2893f304..0000000000 --- a/scripts/build.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -# -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 -# - -set -ex - -function usage() { - echo "Usage: $0 [args]" - echo "" - echo "Arguments:" - echo -e "-v VERSION\t[Required] OpenSearch version." - echo -e "-q QUALIFIER\t[Optional] Version qualifier." - echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'." - echo -e "-p PLATFORM\t[Optional] Platform, ignored." - echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored." - echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'." - echo -e "-h help" -} - -while getopts ":h:v:q:s:o:p:a:" arg; do - case $arg in - h) - usage - exit 1 - ;; - v) - VERSION=$OPTARG - ;; - q) - QUALIFIER=$OPTARG - ;; - s) - SNAPSHOT=$OPTARG - ;; - o) - OUTPUT=$OPTARG - ;; - p) - PLATFORM=$OPTARG - ;; - a) - ARCHITECTURE=$OPTARG - ;; - :) - echo "Error: -${OPTARG} requires an argument" - usage - exit 1 - ;; - ?) - echo "Invalid option: -${arg}" - exit 1 - ;; - esac -done - -if [ -z "$VERSION" ]; then - echo "Error: You must specify the OpenSearch version" - usage - exit 1 -fi - -[[ ! -z "$QUALIFIER" ]] && VERSION=$VERSION-$QUALIFIER -[[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT -[ -z "$OUTPUT" ] && OUTPUT=artifacts - -mkdir -p $OUTPUT - -./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER - -zipPath=$(find . -path \*build/distributions/*.zip) -distributions="$(dirname "${zipPath}")" - -echo "COPY ${distributions}/*.zip" -mkdir -p $OUTPUT/plugins -cp ${distributions}/*.zip ./$OUTPUT/plugins - -./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -mkdir -p $OUTPUT/maven/org/opensearch -cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch From 820c833e22b44b31ff7a9386b15ed8afbd6cbccf Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 16:55:07 -0800 Subject: [PATCH 12/12] Add Support For `TIME` Type in "*_OF_YEAR" Functions (#199) (#1223) (#1258) Added Support And Tests For Time Type in day_of_year, week_of_year, month_of_year Functions Signed-off-by: GabeFernandez310 (cherry picked from commit 6e72f1816efccb3a62f74ed49576d7c4920fcd0e) Co-authored-by: GabeFernandez310 --- .../org/opensearch/sql/expression/DSL.java | 20 +- .../expression/datetime/DateTimeFunction.java | 30 + .../sql/expression/function/FunctionDSL.java | 4 +- .../datetime/DateTimeFunctionTest.java | 558 ++++++++++++------ ...ctionDSLimplWithPropertiesTwoArgsTest.java | 34 ++ docs/user/dql/functions.rst | 17 +- sql/src/main/antlr/OpenSearchSQLParser.g4 | 6 +- 7 files changed, 461 insertions(+), 208 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/function/FunctionDSLimplWithPropertiesTwoArgsTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index bb343cd5f9..611053f0bf 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -335,8 +335,9 @@ public static FunctionExpression dayofyear(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.DAYOFYEAR, expressions); } - public static FunctionExpression day_of_year(Expression... expressions) { - return compile(FunctionProperties.None, BuiltinFunctionName.DAY_OF_YEAR, expressions); + public static FunctionExpression day_of_year( + FunctionProperties functionProperties, Expression... expressions) { + return compile(functionProperties, BuiltinFunctionName.DAY_OF_YEAR, expressions); } public static FunctionExpression day_of_week( @@ -372,8 +373,9 @@ public static FunctionExpression month(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.MONTH, expressions); } - public static FunctionExpression month_of_year(Expression... expressions) { - return compile(FunctionProperties.None, BuiltinFunctionName.MONTH_OF_YEAR, expressions); + public static FunctionExpression month_of_year( + FunctionProperties functionProperties, Expression... expressions) { + return compile(functionProperties, BuiltinFunctionName.MONTH_OF_YEAR, expressions); } public static FunctionExpression monthname(Expression... expressions) { @@ -416,12 +418,14 @@ public static FunctionExpression to_days(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.TO_DAYS, expressions); } - public static FunctionExpression week(Expression... expressions) { - return compile(FunctionProperties.None, BuiltinFunctionName.WEEK, expressions); + public static FunctionExpression week( + FunctionProperties functionProperties, Expression... expressions) { + return compile(functionProperties, BuiltinFunctionName.WEEK, expressions); } - public static FunctionExpression week_of_year(Expression... expressions) { - return compile(FunctionProperties.None, BuiltinFunctionName.WEEK_OF_YEAR, expressions); + public static FunctionExpression week_of_year( + FunctionProperties functionProperties, Expression... expressions) { + return compile(functionProperties, BuiltinFunctionName.WEEK_OF_YEAR, expressions); } public static FunctionExpression year(Expression... expressions) { diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 064d6ea857..68227d1ba4 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -89,6 +89,9 @@ public class DateTimeFunction { // 32536771199.999999, or equivalent '3001-01-18 23:59:59.999999' UTC private static final Double MYSQL_MAX_TIMESTAMP = 32536771200d; + // Mode used for week/week_of_year function by default when no argument is provided + private static final ExprIntegerValue DEFAULT_WEEK_OF_YEAR_MODE = new ExprIntegerValue(0); + /** * Register Date and Time Functions. * @@ -472,6 +475,9 @@ private DefaultFunctionResolver dayOfWeek(FunctionName name) { */ private DefaultFunctionResolver dayOfYear(BuiltinFunctionName dayOfYear) { return define(dayOfYear.getName(), + implWithProperties(nullMissingHandlingWithProperties((functionProperties, arg) + -> DateTimeFunction.dayOfYearToday( + functionProperties.getQueryStartClock())), INTEGER, TIME), impl(nullMissingHandling(DateTimeFunction::exprDayOfYear), INTEGER, DATE), impl(nullMissingHandling(DateTimeFunction::exprDayOfYear), INTEGER, DATETIME), impl(nullMissingHandling(DateTimeFunction::exprDayOfYear), INTEGER, TIMESTAMP), @@ -559,6 +565,9 @@ private DefaultFunctionResolver minute_of_day() { */ private DefaultFunctionResolver month(BuiltinFunctionName month) { return define(month.getName(), + implWithProperties(nullMissingHandlingWithProperties((functionProperties, arg) + -> DateTimeFunction.monthOfYearToday( + functionProperties.getQueryStartClock())), INTEGER, TIME), impl(nullMissingHandling(DateTimeFunction::exprMonth), INTEGER, DATE), impl(nullMissingHandling(DateTimeFunction::exprMonth), INTEGER, DATETIME), impl(nullMissingHandling(DateTimeFunction::exprMonth), INTEGER, TIMESTAMP), @@ -783,10 +792,18 @@ private DefaultFunctionResolver utc_timestamp() { */ private DefaultFunctionResolver week(BuiltinFunctionName week) { return define(week.getName(), + implWithProperties(nullMissingHandlingWithProperties((functionProperties, arg) + -> DateTimeFunction.weekOfYearToday( + DEFAULT_WEEK_OF_YEAR_MODE, + functionProperties.getQueryStartClock())), INTEGER, TIME), impl(nullMissingHandling(DateTimeFunction::exprWeekWithoutMode), INTEGER, DATE), impl(nullMissingHandling(DateTimeFunction::exprWeekWithoutMode), INTEGER, DATETIME), impl(nullMissingHandling(DateTimeFunction::exprWeekWithoutMode), INTEGER, TIMESTAMP), impl(nullMissingHandling(DateTimeFunction::exprWeekWithoutMode), INTEGER, STRING), + implWithProperties(nullMissingHandlingWithProperties((functionProperties, time, modeArg) + -> DateTimeFunction.weekOfYearToday( + modeArg, + functionProperties.getQueryStartClock())), INTEGER, TIME, INTEGER), impl(nullMissingHandling(DateTimeFunction::exprWeek), INTEGER, DATE, INTEGER), impl(nullMissingHandling(DateTimeFunction::exprWeek), INTEGER, DATETIME, INTEGER), impl(nullMissingHandling(DateTimeFunction::exprWeek), INTEGER, TIMESTAMP, INTEGER), @@ -827,6 +844,15 @@ private DefaultFunctionResolver date_format() { ); } + private ExprValue dayOfYearToday(Clock clock) { + return new ExprIntegerValue(LocalDateTime.now(clock).getDayOfYear()); + } + + private ExprValue weekOfYearToday(ExprValue mode, Clock clock) { + return new ExprIntegerValue( + CalendarLookup.getWeekNumber(mode.integerValue(), LocalDateTime.now(clock).toLocalDate())); + } + /** * Day of Week implementation for ExprValue when passing in an arguemt of type TIME. * @@ -1519,6 +1545,10 @@ private ExprValue exprYear(ExprValue date) { return new ExprIntegerValue(date.dateValue().getYear()); } + private ExprValue monthOfYearToday(Clock clock) { + return new ExprIntegerValue(LocalDateTime.now(clock).getMonthValue()); + } + private LocalDateTime formatNow(Clock clock) { return formatNow(clock, 0); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/FunctionDSL.java b/core/src/main/java/org/opensearch/sql/expression/function/FunctionDSL.java index 1bf38f7722..d94d7cdf60 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/FunctionDSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/FunctionDSL.java @@ -141,11 +141,11 @@ public String toString() { * Implementation of a function that takes two arguments, returns a value, and * requires FunctionProperties to complete. * - * @param function {@link ExprValue} based unary function. + * @param function {@link ExprValue} based Binary function. * @param returnType return type. * @param args1Type first argument type. * @param args2Type second argument type. - * @return Unary Function Implementation. + * @return Binary Function Implementation. */ public static SerializableFunction> implWithProperties( diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 7761c1e94c..76f35dc68c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.expression.datetime; +import static java.time.temporal.ChronoField.ALIGNED_WEEK_OF_YEAR; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -604,10 +605,66 @@ public void dayOfYear() { assertEquals(integerValue(220), eval(expression)); } - private void testDayOfYearWithUnderscores(String date, int dayOfYear) { - FunctionExpression expression = DSL.day_of_year(DSL.literal(new ExprDateValue(date))); - assertEquals(INTEGER, expression.type()); - assertEquals(integerValue(dayOfYear), eval(expression)); + private static Stream getTestDataForDayOfYear() { + return Stream.of( + Arguments.of(DSL.literal( + new ExprDateValue("2020-08-07")), + "day_of_year(DATE '2020-08-07')", + 220), + Arguments.of(DSL.literal( + new ExprDatetimeValue("2020-08-07 12:23:34")), + "day_of_year(DATETIME '2020-08-07 12:23:34')", + 220), + Arguments.of(DSL.literal( + new ExprTimestampValue("2020-08-07 12:23:34")), + "day_of_year(TIMESTAMP '2020-08-07 12:23:34')", + 220), + Arguments.of(DSL.literal( + "2020-08-07"), + "day_of_year(\"2020-08-07\")", + 220), + Arguments.of(DSL.literal( + "2020-08-07 01:02:03"), + "day_of_year(\"2020-08-07 01:02:03\")", + 220) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("getTestDataForDayOfYear") + public void dayOfYearWithUnderscores( + LiteralExpression arg, + String expectedString, + int expectedResult) { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + validateStringFormat( + DSL.day_of_year(functionProperties, arg), + expectedString, + expectedResult + ); + } + + @Test + public void testDayOfYearWithTimeType() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + validateStringFormat( + DSL.day_of_year(functionProperties, DSL.literal(new ExprTimeValue("12:23:34"))), + "day_of_year(TIME '12:23:34')", + LocalDate.now(functionProperties.getQueryStartClock()).getDayOfYear()); + } + + public void dayOfYearWithUnderscoresQuery(String date, int dayOfYear) { + FunctionExpression expression = DSL.day_of_year( + functionProperties, + DSL.literal(new ExprDateValue(date))); + assertAll( + () -> assertEquals(INTEGER, expression.type()), + () -> assertEquals(integerValue(dayOfYear), eval(expression)) + ); } @Test @@ -615,18 +672,24 @@ public void dayOfYearWithUnderscoresDifferentArgumentFormats() { lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - FunctionExpression expression1 = DSL.day_of_year(DSL.literal(new ExprDateValue("2020-08-07"))); - FunctionExpression expression2 = DSL.day_of_year(DSL.literal("2020-08-07")); - FunctionExpression expression3 = DSL.day_of_year(DSL.literal("2020-08-07 01:02:03")); + FunctionExpression expression1 = DSL.day_of_year( + functionProperties, + DSL.literal(new ExprDateValue("2020-08-07"))); + FunctionExpression expression2 = DSL.day_of_year( + functionProperties, + DSL.literal("2020-08-07")); + FunctionExpression expression3 = DSL.day_of_year( + functionProperties, + DSL.literal("2020-08-07 01:02:03")); assertAll( - () -> testDayOfYearWithUnderscores("2020-08-07", 220), + () -> dayOfYearWithUnderscoresQuery("2020-08-07", 220), () -> assertEquals("day_of_year(DATE '2020-08-07')", expression1.toString()), - () -> testDayOfYearWithUnderscores("2020-08-07", 220), + () -> dayOfYearWithUnderscoresQuery("2020-08-07", 220), () -> assertEquals("day_of_year(\"2020-08-07\")", expression2.toString()), - () -> testDayOfYearWithUnderscores("2020-08-07 01:02:03", 220), + () -> dayOfYearWithUnderscoresQuery("2020-08-07 01:02:03", 220), () -> assertEquals("day_of_year(\"2020-08-07 01:02:03\")", expression3.toString()) ); } @@ -638,11 +701,11 @@ public void dayOfYearWithUnderscoresCornerCaseDates() { assertAll( //31st of December during non leap year (should be 365) - () -> testDayOfYearWithUnderscores("2019-12-31", 365), + () -> dayOfYearWithUnderscoresQuery("2019-12-31", 365), //Year 1200 - () -> testDayOfYearWithUnderscores("1200-02-28", 59), + () -> dayOfYearWithUnderscoresQuery("1200-02-28", 59), //Year 4000 - () -> testDayOfYearWithUnderscores("4000-02-28", 59) + () -> dayOfYearWithUnderscoresQuery("4000-02-28", 59) ); } @@ -653,26 +716,28 @@ public void dayOfYearWithUnderscoresLeapYear() { assertAll( //28th of Feb - () -> testDayOfYearWithUnderscores("2020-02-28", 59), + () -> dayOfYearWithUnderscoresQuery("2020-02-28", 59), //29th of Feb during leap year - () -> testDayOfYearWithUnderscores("2020-02-29 23:59:59", 60), - () -> testDayOfYearWithUnderscores("2020-02-29", 60), + () -> dayOfYearWithUnderscoresQuery("2020-02-29 23:59:59", 60), + () -> dayOfYearWithUnderscoresQuery("2020-02-29", 60), //1st of March during leap year - () -> testDayOfYearWithUnderscores("2020-03-01 00:00:00", 61), - () -> testDayOfYearWithUnderscores("2020-03-01", 61), + () -> dayOfYearWithUnderscoresQuery("2020-03-01 00:00:00", 61), + () -> dayOfYearWithUnderscoresQuery("2020-03-01", 61), //1st of March during non leap year - () -> testDayOfYearWithUnderscores("2019-03-01", 60), + () -> dayOfYearWithUnderscoresQuery("2019-03-01", 60), //31st of December during leap year (should be 366) - () -> testDayOfYearWithUnderscores("2020-12-31", 366) + () -> dayOfYearWithUnderscoresQuery("2020-12-31", 366) ); } - private void testInvalidDayOfYear(String date) { - FunctionExpression expression = DSL.day_of_year(DSL.literal(new ExprDateValue(date))); + private void invalidDayOfYearQuery(String date) { + FunctionExpression expression = DSL.day_of_year( + functionProperties, + DSL.literal(new ExprDateValue(date))); eval(expression); } @@ -680,15 +745,26 @@ private void testInvalidDayOfYear(String date) { public void invalidDayOfYearArgument() { when(nullRef.type()).thenReturn(DATE); when(missingRef.type()).thenReturn(DATE); - assertEquals(nullValue(), eval(DSL.day_of_year(nullRef))); - assertEquals(missingValue(), eval(DSL.day_of_year(missingRef))); - //29th of Feb non-leapyear - assertThrows(SemanticCheckException.class, () -> testInvalidDayOfYear("2019-02-29")); - //13th month - assertThrows(SemanticCheckException.class, () -> testInvalidDayOfYear("2019-13-15")); - //incorrect format for type - assertThrows(SemanticCheckException.class, () -> testInvalidDayOfYear("asdfasdfasdf")); + assertAll( + () -> assertEquals(nullValue(), eval(DSL.day_of_year(functionProperties, nullRef))), + () -> assertEquals(missingValue(), eval(DSL.day_of_year(functionProperties, missingRef))), + + //29th of Feb non-leapyear + () -> assertThrows( + SemanticCheckException.class, + () -> invalidDayOfYearQuery("2019-02-29")), + + //13th month + () -> assertThrows( + SemanticCheckException.class, + () -> invalidDayOfYearQuery("2019-13-15")), + + //incorrect format for type + () -> assertThrows( + SemanticCheckException.class, + () -> invalidDayOfYearQuery("asdfasdfasdf")) + ); } @Test @@ -989,21 +1065,82 @@ public void month() { assertEquals(integerValue(8), eval(expression)); } - private void testInvalidDates(String date) throws SemanticCheckException { - FunctionExpression expression = DSL.month_of_year(DSL.literal(new ExprDateValue(date))); + private static Stream getTestDataForMonthOfYear() { + return Stream.of( + Arguments.of( + DSL.literal(new ExprDateValue("2020-08-07")), + "month_of_year(DATE '2020-08-07')", + 8), + Arguments.of( + DSL.literal(new ExprDatetimeValue("2020-08-07 12:23:34")), + "month_of_year(DATETIME '2020-08-07 12:23:34')", + 8), + Arguments.of( + DSL.literal(new ExprTimestampValue("2020-08-07 12:23:34")), + "month_of_year(TIMESTAMP '2020-08-07 12:23:34')", + 8), + Arguments.of( + DSL.literal("2020-08-07"), + "month_of_year(\"2020-08-07\")", + 8), + Arguments.of( + DSL.literal("2020-08-07 01:02:03"), + "month_of_year(\"2020-08-07 01:02:03\")", + 8) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("getTestDataForMonthOfYear") + public void monthOfYear(LiteralExpression arg, String expectedString, int expectedResult) { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + validateStringFormat( + DSL.month_of_year(functionProperties, arg), + expectedString, + expectedResult + ); + } + + @Test + public void testMonthOfYearWithTimeType() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + validateStringFormat( + DSL.month_of_year(functionProperties, DSL.literal(new ExprTimeValue("12:23:34"))), + "month_of_year(TIME '12:23:34')", + LocalDate.now(functionProperties.getQueryStartClock()).getMonthValue()); + } + + private void invalidDatesQuery(String date) throws SemanticCheckException { + FunctionExpression expression = DSL.month_of_year( + functionProperties, + DSL.literal(new ExprDateValue(date))); eval(expression); } - @Test void monthOfYearInvalidDates() { + @Test + public void monthOfYearInvalidDates() { when(nullRef.type()).thenReturn(DATE); when(missingRef.type()).thenReturn(DATE); - assertEquals(nullValue(), eval(DSL.month_of_year(nullRef))); - assertEquals(missingValue(), eval(DSL.month_of_year(missingRef))); - assertThrows(SemanticCheckException.class, () -> testInvalidDates("2019-01-50")); - assertThrows(SemanticCheckException.class, () -> testInvalidDates("2019-02-29")); - assertThrows(SemanticCheckException.class, () -> testInvalidDates("2019-02-31")); - assertThrows(SemanticCheckException.class, () -> testInvalidDates("2019-13-05")); + assertAll( + () -> assertEquals(nullValue(), eval(DSL.month_of_year( + functionProperties, + nullRef))), + () -> assertEquals(missingValue(), eval(DSL.month_of_year( + functionProperties, + missingRef))), + () -> assertThrows(SemanticCheckException.class, () -> invalidDatesQuery("2019-01-50")), + () -> assertThrows(SemanticCheckException.class, () -> invalidDatesQuery("2019-02-29")), + () -> assertThrows(SemanticCheckException.class, () -> invalidDatesQuery("2019-02-31")), + () -> assertThrows(SemanticCheckException.class, () -> invalidDatesQuery("2019-13-05")) + ); + + + } @Test @@ -1011,17 +1148,23 @@ public void monthOfYearAlternateArgumentSyntaxes() { lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - FunctionExpression expression = DSL.month_of_year(DSL.literal(new ExprDateValue("2020-08-07"))); + FunctionExpression expression = DSL.month_of_year( + functionProperties, + DSL.literal(new ExprDateValue("2020-08-07"))); assertEquals(INTEGER, expression.type()); assertEquals("month_of_year(DATE '2020-08-07')", expression.toString()); assertEquals(integerValue(8), eval(expression)); - expression = DSL.month_of_year(DSL.literal("2020-08-07")); + expression = DSL.month_of_year( + functionProperties, + DSL.literal("2020-08-07")); assertEquals(INTEGER, expression.type()); assertEquals("month_of_year(\"2020-08-07\")", expression.toString()); assertEquals(integerValue(8), eval(expression)); - expression = DSL.month_of_year(DSL.literal("2020-08-07 01:02:03")); + expression = DSL.month_of_year( + functionProperties, + DSL.literal("2020-08-07 01:02:03")); assertEquals(INTEGER, expression.type()); assertEquals("month_of_year(\"2020-08-07 01:02:03\")", expression.toString()); assertEquals(integerValue(8), eval(expression)); @@ -1325,216 +1468,253 @@ public void timestamp() { assertEquals("timestamp(TIMESTAMP '2020-08-17 01:01:01')", expr.toString()); } - private void testWeek(String date, int mode, int expectedResult) { + private void weekQuery(String date, int mode, int expectedResult) { FunctionExpression expression = DSL - .week(DSL.literal(new ExprDateValue(date)), DSL.literal(mode)); + .week(functionProperties, DSL.literal(new ExprDateValue(date)), DSL.literal(mode)); assertEquals(INTEGER, expression.type()); assertEquals(String.format("week(DATE '%s', %d)", date, mode), expression.toString()); assertEquals(integerValue(expectedResult), eval(expression)); } - private void testNullMissingWeek(ExprCoreType date) { + private void weekOfYearQuery(String date, int mode, int expectedResult) { + FunctionExpression expression = DSL + .week_of_year( + functionProperties, + DSL.literal(new ExprDateValue(date)), DSL.literal(mode)); + assertEquals(INTEGER, expression.type()); + assertEquals(String.format("week_of_year(DATE '%s', %d)", date, mode), expression.toString()); + assertEquals(integerValue(expectedResult), eval(expression)); + } + + private void nullMissingWeekQuery(ExprCoreType date) { when(nullRef.type()).thenReturn(date); when(missingRef.type()).thenReturn(date); - assertEquals(nullValue(), eval(DSL.week(nullRef))); - assertEquals(missingValue(), eval(DSL.week(missingRef))); + assertEquals(nullValue(), eval(DSL.week(functionProperties, nullRef))); + assertEquals(missingValue(), eval(DSL.week(functionProperties, missingRef))); } @Test - public void week() { - testNullMissingWeek(DATE); - testNullMissingWeek(DATETIME); - testNullMissingWeek(TIMESTAMP); - testNullMissingWeek(STRING); + public void testNullMissingWeek() { + nullMissingWeekQuery(DATE); + nullMissingWeekQuery(DATETIME); + nullMissingWeekQuery(TIMESTAMP); + nullMissingWeekQuery(STRING); when(nullRef.type()).thenReturn(INTEGER); when(missingRef.type()).thenReturn(INTEGER); - assertEquals(nullValue(), eval(DSL.week(DSL.literal("2019-01-05"), nullRef))); - assertEquals(missingValue(), eval(DSL.week(DSL.literal("2019-01-05"), missingRef))); + assertEquals(nullValue(), eval(DSL.week( + functionProperties, + DSL.literal("2019-01-05"), nullRef))); + assertEquals(missingValue(), eval(DSL.week( + functionProperties, + DSL.literal("2019-01-05"), missingRef))); when(nullRef.type()).thenReturn(DATE); when(missingRef.type()).thenReturn(INTEGER); - assertEquals(missingValue(), eval(DSL.week(nullRef, missingRef))); + assertEquals(missingValue(), eval(DSL.week( + functionProperties, + nullRef, missingRef))); + } - FunctionExpression expression = DSL - .week(DSL.literal(new ExprTimestampValue("2019-01-05 01:02:03"))); - assertEquals(INTEGER, expression.type()); - assertEquals("week(TIMESTAMP '2019-01-05 01:02:03')", expression.toString()); - assertEquals(integerValue(0), eval(expression)); + private static Stream getTestDataForWeek() { + //Test the behavior of different modes passed into the 'week_of_year' function + return Stream.of( + Arguments.of("2019-01-05", 0, 0), + Arguments.of("2019-01-05", 1, 1), + Arguments.of("2019-01-05", 2, 52), + Arguments.of("2019-01-05", 3, 1), + Arguments.of("2019-01-05", 4, 1), + Arguments.of("2019-01-05", 5, 0), + Arguments.of("2019-01-05", 6, 1), + Arguments.of("2019-01-05", 7, 53), + + Arguments.of("2019-01-06", 0, 1), + Arguments.of("2019-01-06", 1, 1), + Arguments.of("2019-01-06", 2, 1), + Arguments.of("2019-01-06", 3, 1), + Arguments.of("2019-01-06", 4, 2), + Arguments.of("2019-01-06", 5, 0), + Arguments.of("2019-01-06", 6, 2), + Arguments.of("2019-01-06", 7, 53), + + Arguments.of("2019-01-07", 0, 1), + Arguments.of("2019-01-07", 1, 2), + Arguments.of("2019-01-07", 2, 1), + Arguments.of("2019-01-07", 3, 2), + Arguments.of("2019-01-07", 4, 2), + Arguments.of("2019-01-07", 5, 1), + Arguments.of("2019-01-07", 6, 2), + Arguments.of("2019-01-07", 7, 1), + + Arguments.of("2000-01-01", 0, 0), + Arguments.of("2000-01-01", 2, 52), + Arguments.of("1999-12-31", 0, 52) + ); + } - expression = DSL.week(DSL.literal("2019-01-05")); - assertEquals(INTEGER, expression.type()); - assertEquals("week(\"2019-01-05\")", expression.toString()); - assertEquals(integerValue(0), eval(expression)); + @ParameterizedTest(name = "{1}{2}") + @MethodSource("getTestDataForWeek") + public void testWeek(String date, int mode, int expected) { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + weekQuery(date, mode, expected); + weekOfYearQuery(date, mode, expected); + } - expression = DSL.week(DSL.literal("2019-01-05 00:01:00")); - assertEquals(INTEGER, expression.type()); - assertEquals("week(\"2019-01-05 00:01:00\")", expression.toString()); - assertEquals(integerValue(0), eval(expression)); + private void validateStringFormat( + FunctionExpression expr, + String expectedString, + int expectedResult) { + assertAll( + () -> assertEquals(INTEGER, expr.type()), + () -> assertEquals(expectedString, expr.toString()), + () -> assertEquals(integerValue(expectedResult), eval(expr)) + ); + } + + private static Stream getTestDataForWeekFormats() { + return Stream.of( + Arguments.of(DSL.literal(new ExprDateValue("2019-01-05")), + "DATE '2019-01-05'", + 0), + Arguments.of(DSL.literal(new ExprDatetimeValue("2019-01-05 01:02:03")), + "DATETIME '2019-01-05 01:02:03'", + 0), + Arguments.of(DSL.literal(new ExprTimestampValue("2019-01-05 01:02:03")), + "TIMESTAMP '2019-01-05 01:02:03'", + 0), + Arguments.of( + DSL.literal("2019-01-05"), + "\"2019-01-05\"", + 0), + Arguments.of( + DSL.literal("2019-01-05 00:01:00"), + "\"2019-01-05 00:01:00\"", + 0) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("getTestDataForWeekFormats") + public void testWeekFormats( + LiteralExpression arg, + String expectedString, + Integer expectedInteger) { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + validateStringFormat( + DSL.week(functionProperties, arg), + String.format("week(%s)", expectedString), expectedInteger); + validateStringFormat( + DSL.week_of_year(functionProperties, arg), + String.format("week_of_year(%s)", expectedString), expectedInteger); + } + + @Test + public void testWeekOfYearWithTimeType() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - testWeek("2019-01-05", 0, 0); - testWeek("2019-01-05", 1, 1); - testWeek("2019-01-05", 2, 52); - testWeek("2019-01-05", 3, 1); - testWeek("2019-01-05", 4, 1); - testWeek("2019-01-05", 5, 0); - testWeek("2019-01-05", 6, 1); - testWeek("2019-01-05", 7, 53); - - testWeek("2019-01-06", 0, 1); - testWeek("2019-01-06", 1, 1); - testWeek("2019-01-06", 2, 1); - testWeek("2019-01-06", 3, 1); - testWeek("2019-01-06", 4, 2); - testWeek("2019-01-06", 5, 0); - testWeek("2019-01-06", 6, 2); - testWeek("2019-01-06", 7, 53); - - testWeek("2019-01-07", 0, 1); - testWeek("2019-01-07", 1, 2); - testWeek("2019-01-07", 2, 1); - testWeek("2019-01-07", 3, 2); - testWeek("2019-01-07", 4, 2); - testWeek("2019-01-07", 5, 1); - testWeek("2019-01-07", 6, 2); - testWeek("2019-01-07", 7, 1); - - testWeek("2000-01-01", 0, 0); - testWeek("2000-01-01", 2, 52); - testWeek("1999-12-31", 0, 52); + assertAll( + () -> validateStringFormat( + DSL.week( + functionProperties, + DSL.literal(new ExprTimeValue("12:23:34")), + DSL.literal(0)), + "week(TIME '12:23:34', 0)", + LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR)), + + () -> validateStringFormat( + DSL.week_of_year(functionProperties, DSL.literal(new ExprTimeValue("12:23:34"))), + "week_of_year(TIME '12:23:34')", + LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR)) + ); } @Test public void modeInUnsupportedFormat() { - testNullMissingWeek(DATE); + nullMissingWeekQuery(DATE); FunctionExpression expression1 = DSL - .week(DSL.literal(new ExprDateValue("2019-01-05")), DSL.literal(8)); + .week(functionProperties, DSL.literal(new ExprDateValue("2019-01-05")), DSL.literal(8)); SemanticCheckException exception = assertThrows(SemanticCheckException.class, () -> eval(expression1)); assertEquals("mode:8 is invalid, please use mode value between 0-7", exception.getMessage()); FunctionExpression expression2 = DSL - .week(DSL.literal(new ExprDateValue("2019-01-05")), DSL.literal(-1)); + .week(functionProperties, DSL.literal(new ExprDateValue("2019-01-05")), DSL.literal(-1)); exception = assertThrows(SemanticCheckException.class, () -> eval(expression2)); assertEquals("mode:-1 is invalid, please use mode value between 0-7", exception.getMessage()); } - private void testWeekOfYear(String date, int mode, int expectedResult) { - FunctionExpression expression = DSL - .week_of_year(DSL.literal(new ExprDateValue(date)), DSL.literal(mode)); - assertEquals(INTEGER, expression.type()); - assertEquals(String.format("week_of_year(DATE '%s', %d)", date, mode), expression.toString()); - assertEquals(integerValue(expectedResult), eval(expression)); - } - - private void testNullMissingWeekOfYear(ExprCoreType date) { + private void nullMissingWeekOfYearQuery(ExprCoreType date) { when(nullRef.type()).thenReturn(date); when(missingRef.type()).thenReturn(date); - assertEquals(nullValue(), eval(DSL.week_of_year(nullRef))); - assertEquals(missingValue(), eval(DSL.week_of_year(missingRef))); + assertEquals(nullValue(), eval(DSL.week_of_year( + functionProperties, + nullRef))); + assertEquals(missingValue(), eval(DSL.week_of_year( + functionProperties, + missingRef))); } @Test public void testInvalidWeekOfYear() { - testNullMissingWeekOfYear(DATE); - testNullMissingWeekOfYear(DATETIME); - testNullMissingWeekOfYear(TIMESTAMP); - testNullMissingWeekOfYear(STRING); + nullMissingWeekOfYearQuery(DATE); + nullMissingWeekOfYearQuery(DATETIME); + nullMissingWeekOfYearQuery(TIMESTAMP); + nullMissingWeekOfYearQuery(STRING); when(nullRef.type()).thenReturn(INTEGER); when(missingRef.type()).thenReturn(INTEGER); - assertEquals(nullValue(), eval(DSL.week_of_year(DSL.literal("2019-01-05"), nullRef))); - assertEquals(missingValue(), eval(DSL.week_of_year(DSL.literal("2019-01-05"), missingRef))); + assertEquals(nullValue(), eval(DSL.week_of_year( + functionProperties, + DSL.literal("2019-01-05"), nullRef))); + assertEquals(missingValue(), eval(DSL.week_of_year( + functionProperties, + DSL.literal("2019-01-05"), missingRef))); when(nullRef.type()).thenReturn(DATE); when(missingRef.type()).thenReturn(INTEGER); - assertEquals(missingValue(), eval(DSL.week_of_year(nullRef, missingRef))); - - //test invalid month - assertThrows(SemanticCheckException.class, () -> testWeekOfYear("2019-13-05 01:02:03", 0, 0)); - //test invalid day - assertThrows(SemanticCheckException.class, () -> testWeekOfYear("2019-01-50 01:02:03", 0, 0)); - //test invalid leap year - assertThrows(SemanticCheckException.class, () -> testWeekOfYear("2019-02-29 01:02:03", 0, 0)); - } - - @Test - public void testWeekOfYearAlternateArgumentFormats() { - lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); - lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - - FunctionExpression expression = DSL - .week_of_year(DSL.literal(new ExprTimestampValue("2019-01-05 01:02:03"))); - assertEquals(INTEGER, expression.type()); - assertEquals("week_of_year(TIMESTAMP '2019-01-05 01:02:03')", expression.toString()); - assertEquals(integerValue(0), eval(expression)); - - expression = DSL.week_of_year(DSL.literal("2019-01-05")); - assertEquals(INTEGER, expression.type()); - assertEquals("week_of_year(\"2019-01-05\")", expression.toString()); - assertEquals(integerValue(0), eval(expression)); - - expression = DSL.week_of_year(DSL.literal("2019-01-05 00:01:00")); - assertEquals(INTEGER, expression.type()); - assertEquals("week_of_year(\"2019-01-05 00:01:00\")", expression.toString()); - assertEquals(integerValue(0), eval(expression)); - } - - @Test - public void testWeekOfYearDifferentModes() { - lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); - lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - - //Test the behavior of different modes passed into the 'week_of_year' function - testWeekOfYear("2019-01-05", 0, 0); - testWeekOfYear("2019-01-05", 1, 1); - testWeekOfYear("2019-01-05", 2, 52); - testWeekOfYear("2019-01-05", 3, 1); - testWeekOfYear("2019-01-05", 4, 1); - testWeekOfYear("2019-01-05", 5, 0); - testWeekOfYear("2019-01-05", 6, 1); - testWeekOfYear("2019-01-05", 7, 53); - - testWeekOfYear("2019-01-06", 0, 1); - testWeekOfYear("2019-01-06", 1, 1); - testWeekOfYear("2019-01-06", 2, 1); - testWeekOfYear("2019-01-06", 3, 1); - testWeekOfYear("2019-01-06", 4, 2); - testWeekOfYear("2019-01-06", 5, 0); - testWeekOfYear("2019-01-06", 6, 2); - testWeekOfYear("2019-01-06", 7, 53); - - testWeekOfYear("2019-01-07", 0, 1); - testWeekOfYear("2019-01-07", 1, 2); - testWeekOfYear("2019-01-07", 2, 1); - testWeekOfYear("2019-01-07", 3, 2); - testWeekOfYear("2019-01-07", 4, 2); - testWeekOfYear("2019-01-07", 5, 1); - testWeekOfYear("2019-01-07", 6, 2); - testWeekOfYear("2019-01-07", 7, 1); - - testWeekOfYear("2000-01-01", 0, 0); - testWeekOfYear("2000-01-01", 2, 52); - testWeekOfYear("1999-12-31", 0, 52); + assertEquals(missingValue(), eval(DSL.week_of_year( + functionProperties, + nullRef, missingRef))); + assertAll( + //test invalid month + () -> assertThrows( + SemanticCheckException.class, + () -> weekOfYearQuery("2019-13-05 01:02:03", 0, 0)), + //test invalid day + () -> assertThrows( + SemanticCheckException.class, + () -> weekOfYearQuery("2019-01-50 01:02:03", 0, 0)), + //test invalid leap year + () -> assertThrows( + SemanticCheckException.class, + () -> weekOfYearQuery("2019-02-29 01:02:03", 0, 0)) + ); } @Test public void weekOfYearModeInUnsupportedFormat() { - testNullMissingWeekOfYear(DATE); + nullMissingWeekOfYearQuery(DATE); FunctionExpression expression1 = DSL - .week_of_year(DSL.literal(new ExprDateValue("2019-01-05")), DSL.literal(8)); + .week_of_year( + functionProperties, + DSL.literal(new ExprDateValue("2019-01-05")), DSL.literal(8)); SemanticCheckException exception = assertThrows(SemanticCheckException.class, () -> eval(expression1)); assertEquals("mode:8 is invalid, please use mode value between 0-7", exception.getMessage()); FunctionExpression expression2 = DSL - .week_of_year(DSL.literal(new ExprDateValue("2019-01-05")), DSL.literal(-1)); + .week_of_year( + functionProperties, + DSL.literal(new ExprDateValue("2019-01-05")), DSL.literal(-1)); exception = assertThrows(SemanticCheckException.class, () -> eval(expression2)); assertEquals("mode:-1 is invalid, please use mode value between 0-7", exception.getMessage()); diff --git a/core/src/test/java/org/opensearch/sql/expression/function/FunctionDSLimplWithPropertiesTwoArgsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/FunctionDSLimplWithPropertiesTwoArgsTest.java new file mode 100644 index 0000000000..f690485801 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/FunctionDSLimplWithPropertiesTwoArgsTest.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function; + + +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; + +class FunctionDSLimplWithPropertiesTwoArgsTest extends FunctionDSLimplTestBase { + + @Override + SerializableFunction> + getImplementationGenerator() { + SerializableTriFunction functionBody + = (fp, arg1, arg2) -> ANY; + return FunctionDSL.implWithProperties(functionBody, ANY_TYPE, ANY_TYPE, ANY_TYPE); + } + + @Override + List getSampleArguments() { + return List.of(DSL.literal(ANY), DSL.literal(ANY)); + } + + @Override + String getExpected_toString() { + return "sample(ANY, ANY)"; + } +} diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 3dbfedd143..4d0ab13879 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1598,9 +1598,10 @@ Description >>>>>>>>>>> Usage: dayofyear(date) returns the day of the year for date, in the range 1 to 366. +If an argument of type `TIME` is given, the function will use the current date. The function `day_of_year`_ is also provided as an alias. -Argument type: STRING/DATE/DATETIME/TIMESTAMP +Argument type: STRING/DATE/DATETIME/TIME/TIMESTAMP Return type: INTEGER @@ -1637,9 +1638,10 @@ DAY_OF_YEAR Description >>>>>>>>>>> +If an argument of type `TIME` is given, the function will use the current date. This function is an alias to the `dayofyear`_ function -Argument type: STRING/DATE/DATETIME/TIMESTAMP +Argument type: STRING/DATE/DATETIME/TIME/TIMESTAMP Return type: INTEGER @@ -1934,9 +1936,10 @@ Description >>>>>>>>>>> Usage: month(date) returns the month for date, in the range 1 to 12 for January to December. The dates with value 0 such as '0000-00-00' or '2008-00-00' are invalid. -The function month_of_year is also provided as an alias +If an argument of type `TIME` is given, the function will use the current date. +The function `month_of_year`_ is also provided as an alias. -Argument type: STRING/DATE/DATETIME/TIMESTAMP +Argument type: STRING/DATE/DATETIME/TIME/TIMESTAMP Return type: INTEGER @@ -2451,6 +2454,7 @@ Description >>>>>>>>>>> Usage: week(date[, mode]) returns the week number for date. If the mode argument is omitted, the default mode 0 is used. +If an argument of type `TIME` is given, the function will use the current date. The function `week_of_year` is also provided as an alias. .. list-table:: The following table describes how the mode argument works. @@ -2494,7 +2498,7 @@ The function `week_of_year` is also provided as an alias. - 1-53 - with a Monday in this year -Argument type: DATE/DATETIME/TIMESTAMP/STRING +Argument type: DATE/DATETIME/TIME/TIMESTAMP/STRING Return type: INTEGER @@ -2515,8 +2519,9 @@ Description >>>>>>>>>>> The week_of_year function is a synonym for the `week`_ function. +If an argument of type `TIME` is given, the function will use the current date. -Argument type: DATE/DATETIME/TIMESTAMP/STRING +Argument type: DATE/DATETIME/TIME/TIMESTAMP/STRING Return type: INTEGER diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 5f468690d5..ad52de0e39 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -242,14 +242,11 @@ datetimeConstantLiteral : CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP - | DAY_OF_YEAR | LOCALTIME | LOCALTIMESTAMP - | MONTH_OF_YEAR | UTC_TIMESTAMP | UTC_DATE | UTC_TIME - | WEEK_OF_YEAR ; intervalLiteral @@ -441,6 +438,7 @@ dateTimeFunctionName | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR + | DAY_OF_YEAR | DAY_OF_WEEK | FROM_DAYS | FROM_UNIXTIME @@ -453,6 +451,7 @@ dateTimeFunctionName | MINUTE_OF_HOUR | MONTH | MONTHNAME + | MONTH_OF_YEAR | NOW | PERIOD_ADD | PERIOD_DIFF @@ -469,6 +468,7 @@ dateTimeFunctionName | TO_DAYS | UNIX_TIMESTAMP | WEEK + | WEEK_OF_YEAR | YEAR ;