From 2bf7a905a51ea78a807f9f16ccb196555cf469ca Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Mon, 16 Sep 2024 17:25:07 -0700 Subject: [PATCH 01/32] Extend scheduler interface for Multitenancy (#3014) (#3034) Signed-off-by: Louis Chu (cherry picked from commit 37188bdd5154f9a67043dcaa74ddc5587dd52858) --- .../workflows/integ-tests-with-security.yml | 4 +- .../workflows/sql-test-and-build-workflow.yml | 29 ++++++- .../flint/operation/FlintIndexOpAlter.java | 3 +- .../flint/operation/FlintIndexOpDrop.java | 3 +- .../spark/scheduler/AsyncQueryScheduler.java | 26 +++++- .../model/AsyncQuerySchedulerRequest.java | 2 + .../asyncquery/AsyncQueryCoreIntegTest.java | 5 +- .../OpenSearchAsyncQueryScheduler.java | 34 +++++--- .../model/ScheduledAsyncQueryJobRequest.java | 4 +- ...enSearchScheduleQueryJobRequestParser.java | 2 +- .../OpenSearchAsyncQuerySchedulerTest.java | 85 ++++++++++++------- .../job/ScheduledAsyncQueryJobRunnerTest.java | 6 +- .../ScheduledAsyncQueryJobRequestTest.java | 10 +-- 13 files changed, 146 insertions(+), 67 deletions(-) diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 4d19673ad9..4a077c5d66 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -47,7 +47,7 @@ jobs: - name: Upload test reports if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: test-reports-${{ matrix.os }}-${{ matrix.java }} @@ -79,7 +79,7 @@ jobs: - name: Upload test reports if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: test-reports-${{ matrix.os }}-${{ matrix.java }} diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 0b4b2caf5c..13e82745cf 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -76,14 +76,15 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} - name: Upload Artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 + continue-on-error: true with: name: opensearch-sql-ubuntu-latest path: opensearch-sql-builds - name: Upload test reports - if: always() - uses: actions/upload-artifact@v2 + if: ${{ always() }} + uses: actions/upload-artifact@v4 continue-on-error: true with: name: test-reports @@ -130,7 +131,27 @@ jobs: cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ - name: Upload Artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 + continue-on-error: true with: name: opensearch-sql-${{ matrix.entry.os }} path: opensearch-sql-builds + + - name: Upload test reports + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: test-reports-${{ matrix.entry.os }}-${{ matrix.entry.java }} + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** + doctest/build/testclusters/docTestCluster-0/logs/* + integ-test/build/testclusters/*/logs/* diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java index de34803823..596d76c24b 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java @@ -62,7 +62,8 @@ void runOp( this.flintIndexMetadataService.updateIndexToManualRefresh( flintIndexMetadata.getOpensearchIndexName(), flintIndexOptions, asyncQueryRequestContext); if (flintIndexMetadata.getFlintIndexOptions().isExternalScheduler()) { - asyncQueryScheduler.unscheduleJob(flintIndexMetadata.getOpensearchIndexName()); + asyncQueryScheduler.unscheduleJob( + flintIndexMetadata.getOpensearchIndexName(), asyncQueryRequestContext); } else { cancelStreamingJob(flintIndexStateModel); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java index 3fa5423c10..88aca66fef 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java @@ -54,7 +54,8 @@ void runOp( "Performing drop index operation for index: {}", flintIndexMetadata.getOpensearchIndexName()); if (flintIndexMetadata.getFlintIndexOptions().isExternalScheduler()) { - asyncQueryScheduler.unscheduleJob(flintIndexMetadata.getOpensearchIndexName()); + asyncQueryScheduler.unscheduleJob( + flintIndexMetadata.getOpensearchIndexName(), asyncQueryRequestContext); } else { cancelStreamingJob(flintIndexStateModel); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java index 8ac499081e..6d5350821b 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java @@ -1,5 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.spark.scheduler; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest; /** Scheduler interface for scheduling asynchronous query jobs. */ @@ -13,10 +19,13 @@ public interface AsyncQueryScheduler { * task * * @param asyncQuerySchedulerRequest The request containing job configuration details + * @param asyncQueryRequestContext The request context passed to AsyncQueryExecutorService * @throws IllegalArgumentException if a job with the same name already exists * @throws RuntimeException if there's an error during job creation */ - void scheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); + void scheduleJob( + AsyncQuerySchedulerRequest asyncQuerySchedulerRequest, + AsyncQueryRequestContext asyncQueryRequestContext); /** * Updates an existing job with new parameters. This method modifies the configuration of an @@ -26,10 +35,13 @@ public interface AsyncQueryScheduler { * scheduled job - Updating resource allocations for a job * * @param asyncQuerySchedulerRequest The request containing updated job configuration + * @param asyncQueryRequestContext The request context passed to AsyncQueryExecutorService * @throws IllegalArgumentException if the job to be updated doesn't exist * @throws RuntimeException if there's an error during the update process */ - void updateJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); + void updateJob( + AsyncQuerySchedulerRequest asyncQuerySchedulerRequest, + AsyncQueryRequestContext asyncQueryRequestContext); /** * Unschedules a job by marking it as disabled and updating its last update time. This method is @@ -41,8 +53,11 @@ public interface AsyncQueryScheduler { * re-enabling of the job in the future * * @param jobId The unique identifier of the job to unschedule + * @param asyncQueryRequestContext The request context passed to AsyncQueryExecutorService + * @throws IllegalArgumentException if the job to be unscheduled doesn't exist + * @throws RuntimeException if there's an error during the unschedule process */ - void unscheduleJob(String jobId); + void unscheduleJob(String jobId, AsyncQueryRequestContext asyncQueryRequestContext); /** * Removes a job completely from the scheduler. This method permanently deletes the job and all @@ -52,6 +67,9 @@ public interface AsyncQueryScheduler { * created jobs - Freeing up resources by deleting unused job configurations * * @param jobId The unique identifier of the job to remove + * @param asyncQueryRequestContext The request context passed to AsyncQueryExecutorService + * @throws IllegalArgumentException if the job to be removed doesn't exist + * @throws RuntimeException if there's an error during the remove process */ - void removeJob(String jobId); + void removeJob(String jobId, AsyncQueryRequestContext asyncQueryRequestContext); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java index b54e5b30ce..c38d92365a 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java @@ -7,12 +7,14 @@ import java.time.Instant; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import org.opensearch.sql.spark.rest.model.LangType; /** Represents a job request for a scheduled task. */ @Data +@Builder @NoArgsConstructor @AllArgsConstructor public class AsyncQuerySchedulerRequest { diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 52d805dd01..ddadeb65e2 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -230,7 +230,7 @@ public void createDropIndexQueryWithScheduler() { verifyCreateIndexDMLResultCalled(); verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, QueryState.SUCCESS, JobType.BATCH); - verify(asyncQueryScheduler).unscheduleJob(indexName); + verify(asyncQueryScheduler).unscheduleJob(indexName, asyncQueryRequestContext); } @Test @@ -318,8 +318,7 @@ public void createAlterIndexQueryWithScheduler() { FlintIndexOptions flintIndexOptions = flintIndexOptionsArgumentCaptor.getValue(); assertFalse(flintIndexOptions.autoRefresh()); - verify(asyncQueryScheduler).unscheduleJob(indexName); - + verify(asyncQueryScheduler).unscheduleJob(indexName, asyncQueryRequestContext); verifyCreateIndexDMLResultCalled(); verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, QueryState.SUCCESS, JobType.BATCH); } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java index 9ebde4fe83..59bad14320 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java @@ -8,6 +8,7 @@ import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.time.Instant; @@ -35,6 +36,7 @@ import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.scheduler.job.ScheduledAsyncQueryJobRunner; import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest; import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest; @@ -55,7 +57,9 @@ public class OpenSearchAsyncQueryScheduler implements AsyncQueryScheduler { @Override /** Schedules a new job by indexing it into the job index. */ - public void scheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { + public void scheduleJob( + AsyncQuerySchedulerRequest asyncQuerySchedulerRequest, + AsyncQueryRequestContext asyncQueryRequestContext) { ScheduledAsyncQueryJobRequest request = ScheduledAsyncQueryJobRequest.fromAsyncQuerySchedulerRequest(asyncQuerySchedulerRequest); if (!this.clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)) { @@ -87,15 +91,18 @@ public void scheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { /** Unschedules a job by marking it as disabled and updating its last update time. */ @Override - public void unscheduleJob(String jobId) { - ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() - .jobId(jobId) - .enabled(false) - .lastUpdateTime(Instant.now()) - .build(); + public void unscheduleJob(String jobId, AsyncQueryRequestContext asyncQueryRequestContext) { + if (Strings.isNullOrEmpty(jobId)) { + throw new IllegalArgumentException("JobId cannot be null or empty"); + } try { - updateJob(request); + AsyncQuerySchedulerRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(jobId) + .enabled(false) + .lastUpdateTime(Instant.now()) + .build(); + updateJob(request, asyncQueryRequestContext); LOG.info("Unscheduled job for jobId: {}", jobId); } catch (IllegalStateException | DocumentMissingException e) { LOG.error("Failed to unschedule job: {}", jobId, e); @@ -105,7 +112,9 @@ public void unscheduleJob(String jobId) { /** Updates an existing job with new parameters. */ @Override @SneakyThrows - public void updateJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { + public void updateJob( + AsyncQuerySchedulerRequest asyncQuerySchedulerRequest, + AsyncQueryRequestContext asyncQueryRequestContext) { ScheduledAsyncQueryJobRequest request = ScheduledAsyncQueryJobRequest.fromAsyncQuerySchedulerRequest(asyncQuerySchedulerRequest); assertIndexExists(); @@ -134,8 +143,11 @@ public void updateJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { /** Removes a job by deleting its document from the index. */ @Override - public void removeJob(String jobId) { + public void removeJob(String jobId, AsyncQueryRequestContext asyncQueryRequestContext) { assertIndexExists(); + if (Strings.isNullOrEmpty(jobId)) { + throw new IllegalArgumentException("JobId cannot be null or empty"); + } DeleteRequest deleteRequest = new DeleteRequest(SCHEDULER_INDEX_NAME, jobId); deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); ActionFuture deleteResponseActionFuture = client.delete(deleteRequest); diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java index 9b85a11888..48aa52a3ce 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java @@ -38,7 +38,7 @@ public class ScheduledAsyncQueryJobRequest extends AsyncQuerySchedulerRequest public static final String ENABLED_FIELD = "enabled"; private final Schedule schedule; - @Builder + @Builder(builderMethodName = "scheduledAsyncQueryJobRequestBuilder") public ScheduledAsyncQueryJobRequest( String accountId, String jobId, @@ -139,7 +139,7 @@ public static ScheduledAsyncQueryJobRequest fromAsyncQuerySchedulerRequest( AsyncQuerySchedulerRequest request) { Instant updateTime = request.getLastUpdateTime() != null ? request.getLastUpdateTime() : Instant.now(); - return ScheduledAsyncQueryJobRequest.builder() + return ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId(request.getAccountId()) .jobId(request.getJobId()) .dataSource(request.getDataSource()) diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java index 9e33ef0248..a824797066 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java @@ -30,7 +30,7 @@ private static Instant parseInstantValue(XContentParser parser) throws IOExcepti public static ScheduledJobParser getJobParser() { return (parser, id, jobDocVersion) -> { ScheduledAsyncQueryJobRequest.ScheduledAsyncQueryJobRequestBuilder builder = - ScheduledAsyncQueryJobRequest.builder(); + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder(); XContentParserUtils.ensureExpectedToken( XContentParser.Token.START_OBJECT, parser.nextToken(), parser); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java index a4a6eb6471..d6e672f7a2 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java @@ -16,7 +16,6 @@ import static org.mockito.Mockito.when; import static org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler.SCHEDULER_INDEX_NAME; -import java.io.IOException; import java.time.Instant; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -43,6 +42,8 @@ import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest; import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest; public class OpenSearchAsyncQuerySchedulerTest { @@ -57,6 +58,8 @@ public class OpenSearchAsyncQuerySchedulerTest { @Mock(answer = Answers.RETURNS_DEEP_STUBS) private ClusterService clusterService; + @Mock private AsyncQueryRequestContext context; + @Mock private ActionFuture indexResponseActionFuture; @Mock private ActionFuture updateResponseActionFuture; @@ -92,12 +95,12 @@ public void testScheduleJob() { when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.CREATED); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); - scheduler.scheduleJob(request); + scheduler.scheduleJob(request, context); // Verify index created verify(client.admin().indices(), times(1)).create(ArgumentMatchers.any()); @@ -116,7 +119,7 @@ public void testScheduleJobWithExistingJob() { .thenReturn(Boolean.TRUE); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -127,7 +130,7 @@ public void testScheduleJobWithExistingJob() { assertThrows( IllegalArgumentException.class, () -> { - scheduler.scheduleJob(request); + scheduler.scheduleJob(request, context); }); verify(client, times(1)).index(ArgumentCaptor.forClass(IndexRequest.class).capture()); @@ -145,24 +148,24 @@ public void testScheduleJobWithExceptions() { when(client.index(any(IndexRequest.class))).thenThrow(new RuntimeException("Test exception")); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); - assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request)); + assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request, context)); when(client.index(any(IndexRequest.class))).thenReturn(indexResponseActionFuture); when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); RuntimeException exception = - assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request)); + assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request, context)); assertEquals("Schedule job failed with result : not_found", exception.getMessage()); } @Test - public void testUnscheduleJob() throws IOException { + public void testUnscheduleJob() { when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); @@ -170,7 +173,7 @@ public void testUnscheduleJob() throws IOException { when(client.update(any(UpdateRequest.class))).thenReturn(updateResponseActionFuture); - scheduler.unscheduleJob(TEST_JOB_ID); + scheduler.unscheduleJob(TEST_JOB_ID, context); ArgumentCaptor captor = ArgumentCaptor.forClass(UpdateRequest.class); verify(client).update(captor.capture()); @@ -183,7 +186,7 @@ public void testUnscheduleJob() throws IOException { captor = ArgumentCaptor.forClass(UpdateRequest.class); when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.NOOP); - scheduler.unscheduleJob(TEST_JOB_ID); + scheduler.unscheduleJob(TEST_JOB_ID, context); verify(client, times(2)).update(captor.capture()); capturedRequest = captor.getValue(); @@ -191,20 +194,29 @@ public void testUnscheduleJob() throws IOException { assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, capturedRequest.getRefreshPolicy()); } + @Test + public void testUnscheduleJobInvalidJobId() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> scheduler.unscheduleJob("", context)); + assertEquals("JobId cannot be null or empty", exception.getMessage()); + } + @Test public void testUnscheduleJobWithIndexNotFound() { when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); - scheduler.unscheduleJob(TEST_JOB_ID); + scheduler.unscheduleJob(TEST_JOB_ID, context); // Verify that no update operation was performed verify(client, never()).update(any(UpdateRequest.class)); } @Test - public void testUpdateJob() throws IOException { + public void testUpdateJob() { ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -216,7 +228,7 @@ public void testUpdateJob() throws IOException { when(client.update(any(UpdateRequest.class))).thenReturn(updateResponseActionFuture); - scheduler.updateJob(request); + scheduler.updateJob(request, context); ArgumentCaptor captor = ArgumentCaptor.forClass(UpdateRequest.class); verify(client).update(captor.capture()); @@ -229,20 +241,20 @@ public void testUpdateJob() throws IOException { @Test public void testUpdateJobWithIndexNotFound() { ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); - assertThrows(IllegalStateException.class, () -> scheduler.updateJob(request)); + assertThrows(IllegalStateException.class, () -> scheduler.updateJob(request, context)); } @Test public void testUpdateJobWithExceptions() { ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -255,7 +267,7 @@ public void testUpdateJobWithExceptions() { assertThrows( IllegalArgumentException.class, () -> { - scheduler.updateJob(request); + scheduler.updateJob(request, context); }); assertEquals("Job: testJob doesn't exist", exception1.getMessage()); @@ -266,7 +278,7 @@ public void testUpdateJobWithExceptions() { assertThrows( RuntimeException.class, () -> { - scheduler.updateJob(request); + scheduler.updateJob(request, context); }); assertEquals("java.lang.RuntimeException: Test exception", exception2.getMessage()); @@ -276,7 +288,7 @@ public void testUpdateJobWithExceptions() { when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); RuntimeException exception = - assertThrows(RuntimeException.class, () -> scheduler.updateJob(request)); + assertThrows(RuntimeException.class, () -> scheduler.updateJob(request, context)); assertEquals("Update job failed with result : not_found", exception.getMessage()); } @@ -290,7 +302,7 @@ public void testRemoveJob() { when(client.delete(any(DeleteRequest.class))).thenReturn(deleteResponseActionFuture); - scheduler.removeJob(TEST_JOB_ID); + scheduler.removeJob(TEST_JOB_ID, context); ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteRequest.class); verify(client).delete(captor.capture()); @@ -304,7 +316,18 @@ public void testRemoveJob() { public void testRemoveJobWithIndexNotFound() { when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); - assertThrows(IllegalStateException.class, () -> scheduler.removeJob(TEST_JOB_ID)); + AsyncQuerySchedulerRequest request = + AsyncQuerySchedulerRequest.builder().jobId(TEST_JOB_ID).build(); + assertThrows(IllegalStateException.class, () -> scheduler.removeJob(TEST_JOB_ID, context)); + } + + @Test + public void testRemoveJobInvalidJobId() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> scheduler.removeJob("", context)); + assertEquals("JobId cannot be null or empty", exception.getMessage()); } @Test @@ -351,13 +374,14 @@ public void testCreateAsyncQuerySchedulerIndexFailure() { .thenReturn(new CreateIndexResponse(false, false, SCHEDULER_INDEX_NAME)); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); RuntimeException runtimeException = - Assertions.assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request)); + Assertions.assertThrows( + RuntimeException.class, () -> scheduler.scheduleJob(request, context)); Assertions.assertEquals( "Internal server error while creating .async-query-scheduler index: Index creation is not" + " acknowledged.", @@ -367,7 +391,7 @@ public void testCreateAsyncQuerySchedulerIndexFailure() { @Test public void testUpdateJobNotFound() { ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -381,7 +405,7 @@ public void testUpdateJobNotFound() { assertThrows( IllegalArgumentException.class, () -> { - scheduler.updateJob(request); + scheduler.updateJob(request, context); }); assertEquals("Job: testJob doesn't exist", exception.getMessage()); @@ -401,7 +425,7 @@ public void testRemoveJobNotFound() { assertThrows( IllegalArgumentException.class, () -> { - scheduler.removeJob(TEST_JOB_ID); + scheduler.removeJob(TEST_JOB_ID, context); }); assertEquals("Job : testJob doesn't exist", exception.getMessage()); @@ -413,7 +437,7 @@ public void testRemoveJobWithExceptions() { when(client.delete(any(DeleteRequest.class))).thenThrow(new RuntimeException("Test exception")); - assertThrows(RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID)); + assertThrows(RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID, context)); DeleteResponse deleteResponse = mock(DeleteResponse.class); when(client.delete(any(DeleteRequest.class))).thenReturn(deleteResponseActionFuture); @@ -421,7 +445,8 @@ public void testRemoveJobWithExceptions() { when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOOP); RuntimeException runtimeException = - Assertions.assertThrows(RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID)); + Assertions.assertThrows( + RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID, context)); Assertions.assertEquals("Remove job failed with result : noop", runtimeException.getMessage()); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java index cba8d43a2a..fdfb138ddb 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java @@ -72,7 +72,7 @@ public void testRunJobWithCorrectParameter() { spyJobRunner.loadJobResource(client, clusterService, threadPool, asyncQueryExecutorService); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId("testJob") .lastUpdateTime(Instant.now()) .lockDurationSeconds(10L) @@ -123,7 +123,7 @@ public void testDoRefreshThrowsException() { spyJobRunner.loadJobResource(client, clusterService, threadPool, asyncQueryExecutorService); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId("testJob") .lastUpdateTime(Instant.now()) .lockDurationSeconds(10L) @@ -158,7 +158,7 @@ public void testDoRefreshThrowsException() { @Test public void testRunJobWithUninitializedServices() { ScheduledAsyncQueryJobRequest jobParameter = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId("testJob") .lastUpdateTime(Instant.now()) .build(); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java index 85d1948dc3..edf8379195 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java @@ -28,7 +28,7 @@ public void testBuilderAndGetterMethods() { IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); ScheduledAsyncQueryJobRequest jobRequest = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("testAccount") .jobId("testJob") .dataSource("testDataSource") @@ -62,7 +62,7 @@ public void testToXContent() throws IOException { IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("testAccount") .jobId("testJob") .dataSource("testDataSource") @@ -146,7 +146,7 @@ public void testEqualsAndHashCode() { IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); ScheduledAsyncQueryJobRequest request1 = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("testAccount") .jobId("testJob") .dataSource("testDataSource") @@ -172,7 +172,7 @@ public void testEqualsAndHashCode() { assertTrue(toString.contains("jitter=0.1")); ScheduledAsyncQueryJobRequest request2 = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("testAccount") .jobId("testJob") .dataSource("testDataSource") @@ -190,7 +190,7 @@ public void testEqualsAndHashCode() { assertEquals(request1.hashCode(), request2.hashCode()); ScheduledAsyncQueryJobRequest request3 = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("differentAccount") .jobId("testJob") .dataSource("testDataSource") From a20f655ef3d03405e8427aacef2d5dfebcab48c3 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:08:35 -0700 Subject: [PATCH 02/32] [Backport 2.x] Upgrade org.bouncycastle:bcprov-jdk18on to 1.78.1 (#3020) Signed-off-by: Peng Huo Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] Co-authored-by: Peng Huo --- .../workflows/integ-tests-with-security.yml | 4 +-- .../workflows/sql-test-and-build-workflow.yml | 29 +++---------------- datasources/build.gradle | 7 ++++- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 4a077c5d66..753aff410e 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -47,7 +47,7 @@ jobs: - name: Upload test reports if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 continue-on-error: true with: name: test-reports-${{ matrix.os }}-${{ matrix.java }} @@ -79,7 +79,7 @@ jobs: - name: Upload test reports if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 continue-on-error: true with: name: test-reports-${{ matrix.os }}-${{ matrix.java }} diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 13e82745cf..92089589e0 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -76,15 +76,14 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} - name: Upload Artifacts - uses: actions/upload-artifact@v4 - continue-on-error: true + uses: actions/upload-artifact@v3 with: name: opensearch-sql-ubuntu-latest path: opensearch-sql-builds - name: Upload test reports - if: ${{ always() }} - uses: actions/upload-artifact@v4 + if: always() + uses: actions/upload-artifact@v3 continue-on-error: true with: name: test-reports @@ -131,27 +130,7 @@ jobs: cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ - name: Upload Artifacts - uses: actions/upload-artifact@v4 - continue-on-error: true + uses: actions/upload-artifact@v3 with: name: opensearch-sql-${{ matrix.entry.os }} path: opensearch-sql-builds - - - name: Upload test reports - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: actions/upload-artifact@v4 - continue-on-error: true - with: - name: test-reports-${{ matrix.entry.os }}-${{ matrix.entry.java }} - path: | - sql/build/reports/** - ppl/build/reports/** - core/build/reports/** - common/build/reports/** - opensearch/build/reports/** - integ-test/build/reports/** - protocol/build/reports/** - legacy/build/reports/** - plugin/build/reports/** - doctest/build/testclusters/docTestCluster-0/logs/* - integ-test/build/testclusters/*/logs/* diff --git a/datasources/build.gradle b/datasources/build.gradle index 9bd233e1f9..9456141e80 100644 --- a/datasources/build.gradle +++ b/datasources/build.gradle @@ -22,7 +22,12 @@ dependencies { implementation group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" implementation group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}" implementation group: 'commons-io', name: 'commons-io', version: '2.8.0' - implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.1' + // FIXME. upgrade aws-encryption-sdk-java once the bouncycastle dependency update to 1.78. + implementation ('com.amazonaws:aws-encryption-sdk-java:2.4.1') { + exclude group: 'org.bouncycastle', module: 'bcprov-ext-jdk18on' + } + // Use OpenSearch bouncycastle version. https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/version.properties + implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}" implementation group: 'commons-validator', name: 'commons-validator', version: '1.7' testImplementation group: 'junit', name: 'junit', version: '4.13.2' From 5ed2a28f60a27c0282c4a42b270deafa27c4c25a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 20:55:02 -0700 Subject: [PATCH 03/32] Implement SQL validation based on grammar element (#3039) (#3044) * Implement SQL validation based on grammar element * Add function types * fix style * Add security lake * Add File support * Integrate into SparkQueryDispatcher * Fix style * Add tests * Integration * Add comments * Address comments * Allow join types for now * Fix style * Fix coverage check --------- (cherry picked from commit a87893ac6771cb8739b82cacf8721d0dd7d1cbe3) Signed-off-by: Tomoyuki Morita Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- async-query-core/build.gradle | 3 +- .../dispatcher/SparkQueryDispatcher.java | 11 +- .../sql/spark/utils/SQLQueryUtils.java | 67 +- .../DefaultGrammarElementValidator.java | 13 + .../DenyListGrammarElementValidator.java | 19 + .../sql/spark/validator/FunctionType.java | 436 +++++++++++++ .../sql/spark/validator/GrammarElement.java | 89 +++ .../validator/GrammarElementValidator.java | 15 + .../GrammarElementValidatorProvider.java | 22 + .../S3GlueGrammarElementValidator.java | 71 ++ .../validator/SQLQueryValidationVisitor.java | 609 ++++++++++++++++++ .../spark/validator/SQLQueryValidator.java | 39 ++ .../SecurityLakeGrammarElementValidator.java | 113 ++++ .../asyncquery/AsyncQueryCoreIntegTest.java | 15 +- .../dispatcher/SparkQueryDispatcherTest.java | 39 +- .../sql/spark/utils/SQLQueryUtilsTest.java | 102 --- .../sql/spark/validator/FunctionTypeTest.java | 47 ++ .../GrammarElementValidatorProviderTest.java | 39 ++ .../validator/SQLQueryValidatorTest.java | 600 +++++++++++++++++ .../config/AsyncExecutorServiceModule.java | 24 +- ...AsyncQueryExecutorServiceImplSpecTest.java | 2 +- .../AsyncQueryExecutorServiceSpec.java | 12 +- 22 files changed, 2195 insertions(+), 192 deletions(-) create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/DefaultGrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/DenyListGrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/FunctionType.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProvider.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/validator/FunctionTypeTest.java create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index 1de6cb3105..deba81735d 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -122,7 +122,8 @@ jacocoTestCoverageVerification { 'org.opensearch.sql.spark.flint.*', 'org.opensearch.sql.spark.flint.operation.*', 'org.opensearch.sql.spark.rest.*', - 'org.opensearch.sql.spark.utils.SQLQueryUtils.*' + 'org.opensearch.sql.spark.utils.SQLQueryUtils.*', + 'org.opensearch.sql.spark.validator.SQLQueryValidationVisitor' ] limit { counter = 'LINE' diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index 732f5f71ab..ff8c8d1fe8 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -6,7 +6,6 @@ package org.opensearch.sql.spark.dispatcher; import java.util.HashMap; -import java.util.List; import java.util.Map; import lombok.AllArgsConstructor; import org.jetbrains.annotations.NotNull; @@ -24,6 +23,7 @@ import org.opensearch.sql.spark.execution.session.SessionManager; import org.opensearch.sql.spark.rest.model.LangType; import org.opensearch.sql.spark.utils.SQLQueryUtils; +import org.opensearch.sql.spark.validator.SQLQueryValidator; /** This class takes care of understanding query and dispatching job query to emr serverless. */ @AllArgsConstructor @@ -38,6 +38,7 @@ public class SparkQueryDispatcher { private final SessionManager sessionManager; private final QueryHandlerFactory queryHandlerFactory; private final QueryIdProvider queryIdProvider; + private final SQLQueryValidator sqlQueryValidator; public DispatchQueryResponse dispatch( DispatchQueryRequest dispatchQueryRequest, @@ -54,13 +55,7 @@ public DispatchQueryResponse dispatch( dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); } - List validationErrors = - SQLQueryUtils.validateSparkSqlQuery( - dataSourceService.getDataSource(dispatchQueryRequest.getDatasource()), query); - if (!validationErrors.isEmpty()) { - throw new IllegalArgumentException( - "Query is not allowed: " + String.join(", ", validationErrors)); - } + sqlQueryValidator.validate(query, dataSourceMetadata.getConnector()); } return handleDefaultQuery(dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java index 7550de2f1e..3ba9c23ed7 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java @@ -5,8 +5,6 @@ package org.opensearch.sql.spark.utils; -import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -20,8 +18,6 @@ import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; import org.opensearch.sql.common.antlr.SyntaxCheckException; -import org.opensearch.sql.datasource.model.DataSource; -import org.opensearch.sql.datasource.model.DataSourceType; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsBaseVisitor; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsLexer; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsParser; @@ -84,71 +80,12 @@ public static boolean isFlintExtensionQuery(String sqlQuery) { } } - public static List validateSparkSqlQuery(DataSource datasource, String sqlQuery) { + public static SqlBaseParser getBaseParser(String sqlQuery) { SqlBaseParser sqlBaseParser = new SqlBaseParser( new CommonTokenStream(new SqlBaseLexer(new CaseInsensitiveCharStream(sqlQuery)))); sqlBaseParser.addErrorListener(new SyntaxAnalysisErrorListener()); - try { - SqlBaseValidatorVisitor sqlParserBaseVisitor = getSparkSqlValidatorVisitor(datasource); - StatementContext statement = sqlBaseParser.statement(); - sqlParserBaseVisitor.visit(statement); - return sqlParserBaseVisitor.getValidationErrors(); - } catch (SyntaxCheckException e) { - logger.error( - String.format( - "Failed to parse sql statement context while validating sql query %s", sqlQuery), - e); - return Collections.emptyList(); - } - } - - private SqlBaseValidatorVisitor getSparkSqlValidatorVisitor(DataSource datasource) { - if (datasource != null - && datasource.getConnectorType() != null - && datasource.getConnectorType().equals(DataSourceType.SECURITY_LAKE)) { - return new SparkSqlSecurityLakeValidatorVisitor(); - } else { - return new SparkSqlValidatorVisitor(); - } - } - - /** - * A base class extending SqlBaseParserBaseVisitor for validating Spark Sql Queries. The class - * supports accumulating validation errors on visiting sql statement - */ - @Getter - private static class SqlBaseValidatorVisitor extends SqlBaseParserBaseVisitor { - private final List validationErrors = new ArrayList<>(); - } - - /** A generic validator impl for Spark Sql Queries */ - private static class SparkSqlValidatorVisitor extends SqlBaseValidatorVisitor { - @Override - public Void visitCreateFunction(SqlBaseParser.CreateFunctionContext ctx) { - getValidationErrors().add("Creating user-defined functions is not allowed"); - return super.visitCreateFunction(ctx); - } - } - - /** A validator impl specific to Security Lake for Spark Sql Queries */ - private static class SparkSqlSecurityLakeValidatorVisitor extends SqlBaseValidatorVisitor { - - public SparkSqlSecurityLakeValidatorVisitor() { - // only select statement allowed. hence we add the validation error to all types of statements - // by default - // and remove the validation error only for select statement. - getValidationErrors() - .add( - "Unsupported sql statement for security lake data source. Only select queries are" - + " allowed"); - } - - @Override - public Void visitStatementDefault(SqlBaseParser.StatementDefaultContext ctx) { - getValidationErrors().clear(); - return super.visitStatementDefault(ctx); - } + return sqlBaseParser; } public static class SparkSqlTableNameVisitor extends SqlBaseParserBaseVisitor { diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DefaultGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DefaultGrammarElementValidator.java new file mode 100644 index 0000000000..ddd0a1d094 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DefaultGrammarElementValidator.java @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +public class DefaultGrammarElementValidator implements GrammarElementValidator { + @Override + public boolean isValid(GrammarElement element) { + return true; + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DenyListGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DenyListGrammarElementValidator.java new file mode 100644 index 0000000000..514e2c8ad8 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DenyListGrammarElementValidator.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import java.util.Set; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class DenyListGrammarElementValidator implements GrammarElementValidator { + private final Set denyList; + + @Override + public boolean isValid(GrammarElement element) { + return !denyList.contains(element); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/FunctionType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/FunctionType.java new file mode 100644 index 0000000000..da3760efd6 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/FunctionType.java @@ -0,0 +1,436 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; + +/** + * Enum for defining and looking up SQL function type based on its name. Unknown one will be + * considered as UDF (User Defined Function) + */ +@AllArgsConstructor +public enum FunctionType { + AGGREGATE("Aggregate"), + WINDOW("Window"), + ARRAY("Array"), + MAP("Map"), + DATE_TIMESTAMP("Date and Timestamp"), + JSON("JSON"), + MATH("Math"), + STRING("String"), + CONDITIONAL("Conditional"), + BITWISE("Bitwise"), + CONVERSION("Conversion"), + PREDICATE("Predicate"), + CSV("CSV"), + MISC("Misc"), + GENERATOR("Generator"), + UDF("User Defined Function"); + + private final String name; + + private static final Map> FUNCTION_TYPE_TO_FUNCTION_NAMES_MAP = + ImmutableMap.>builder() + .put( + AGGREGATE, + Set.of( + "any", + "any_value", + "approx_count_distinct", + "approx_percentile", + "array_agg", + "avg", + "bit_and", + "bit_or", + "bit_xor", + "bitmap_construct_agg", + "bitmap_or_agg", + "bool_and", + "bool_or", + "collect_list", + "collect_set", + "corr", + "count", + "count_if", + "count_min_sketch", + "covar_pop", + "covar_samp", + "every", + "first", + "first_value", + "grouping", + "grouping_id", + "histogram_numeric", + "hll_sketch_agg", + "hll_union_agg", + "kurtosis", + "last", + "last_value", + "max", + "max_by", + "mean", + "median", + "min", + "min_by", + "mode", + "percentile", + "percentile_approx", + "regr_avgx", + "regr_avgy", + "regr_count", + "regr_intercept", + "regr_r2", + "regr_slope", + "regr_sxx", + "regr_sxy", + "regr_syy", + "skewness", + "some", + "std", + "stddev", + "stddev_pop", + "stddev_samp", + "sum", + "try_avg", + "try_sum", + "var_pop", + "var_samp", + "variance")) + .put( + WINDOW, + Set.of( + "cume_dist", + "dense_rank", + "lag", + "lead", + "nth_value", + "ntile", + "percent_rank", + "rank", + "row_number")) + .put( + ARRAY, + Set.of( + "array", + "array_append", + "array_compact", + "array_contains", + "array_distinct", + "array_except", + "array_insert", + "array_intersect", + "array_join", + "array_max", + "array_min", + "array_position", + "array_prepend", + "array_remove", + "array_repeat", + "array_union", + "arrays_overlap", + "arrays_zip", + "flatten", + "get", + "sequence", + "shuffle", + "slice", + "sort_array")) + .put( + MAP, + Set.of( + "element_at", + "map", + "map_concat", + "map_contains_key", + "map_entries", + "map_from_arrays", + "map_from_entries", + "map_keys", + "map_values", + "str_to_map", + "try_element_at")) + .put( + DATE_TIMESTAMP, + Set.of( + "add_months", + "convert_timezone", + "curdate", + "current_date", + "current_timestamp", + "current_timezone", + "date_add", + "date_diff", + "date_format", + "date_from_unix_date", + "date_part", + "date_sub", + "date_trunc", + "dateadd", + "datediff", + "datepart", + "day", + "dayofmonth", + "dayofweek", + "dayofyear", + "extract", + "from_unixtime", + "from_utc_timestamp", + "hour", + "last_day", + "localtimestamp", + "make_date", + "make_dt_interval", + "make_interval", + "make_timestamp", + "make_timestamp_ltz", + "make_timestamp_ntz", + "make_ym_interval", + "minute", + "month", + "months_between", + "next_day", + "now", + "quarter", + "second", + "session_window", + "timestamp_micros", + "timestamp_millis", + "timestamp_seconds", + "to_date", + "to_timestamp", + "to_timestamp_ltz", + "to_timestamp_ntz", + "to_unix_timestamp", + "to_utc_timestamp", + "trunc", + "try_to_timestamp", + "unix_date", + "unix_micros", + "unix_millis", + "unix_seconds", + "unix_timestamp", + "weekday", + "weekofyear", + "window", + "window_time", + "year")) + .put( + JSON, + Set.of( + "from_json", + "get_json_object", + "json_array_length", + "json_object_keys", + "json_tuple", + "schema_of_json", + "to_json")) + .put( + MATH, + Set.of( + "abs", + "acos", + "acosh", + "asin", + "asinh", + "atan", + "atan2", + "atanh", + "bin", + "bround", + "cbrt", + "ceil", + "ceiling", + "conv", + "cos", + "cosh", + "cot", + "csc", + "degrees", + "e", + "exp", + "expm1", + "factorial", + "floor", + "greatest", + "hex", + "hypot", + "least", + "ln", + "log", + "log10", + "log1p", + "log2", + "negative", + "pi", + "pmod", + "positive", + "pow", + "power", + "radians", + "rand", + "randn", + "random", + "rint", + "round", + "sec", + "shiftleft", + "sign", + "signum", + "sin", + "sinh", + "sqrt", + "tan", + "tanh", + "try_add", + "try_divide", + "try_multiply", + "try_subtract", + "unhex", + "width_bucket")) + .put( + STRING, + Set.of( + "ascii", + "base64", + "bit_length", + "btrim", + "char", + "char_length", + "character_length", + "chr", + "concat", + "concat_ws", + "contains", + "decode", + "elt", + "encode", + "endswith", + "find_in_set", + "format_number", + "format_string", + "initcap", + "instr", + "lcase", + "left", + "len", + "length", + "levenshtein", + "locate", + "lower", + "lpad", + "ltrim", + "luhn_check", + "mask", + "octet_length", + "overlay", + "position", + "printf", + "regexp_count", + "regexp_extract", + "regexp_extract_all", + "regexp_instr", + "regexp_replace", + "regexp_substr", + "repeat", + "replace", + "right", + "rpad", + "rtrim", + "sentences", + "soundex", + "space", + "split", + "split_part", + "startswith", + "substr", + "substring", + "substring_index", + "to_binary", + "to_char", + "to_number", + "to_varchar", + "translate", + "trim", + "try_to_binary", + "try_to_number", + "ucase", + "unbase64", + "upper")) + .put(CONDITIONAL, Set.of("coalesce", "if", "ifnull", "nanvl", "nullif", "nvl", "nvl2")) + .put( + BITWISE, Set.of("bit_count", "bit_get", "getbit", "shiftright", "shiftrightunsigned")) + .put( + CONVERSION, + Set.of( + "bigint", + "binary", + "boolean", + "cast", + "date", + "decimal", + "double", + "float", + "int", + "smallint", + "string", + "timestamp", + "tinyint")) + .put(PREDICATE, Set.of("isnan", "isnotnull", "isnull", "regexp", "regexp_like", "rlike")) + .put(CSV, Set.of("from_csv", "schema_of_csv", "to_csv")) + .put( + MISC, + Set.of( + "aes_decrypt", + "aes_encrypt", + "assert_true", + "bitmap_bit_position", + "bitmap_bucket_number", + "bitmap_count", + "current_catalog", + "current_database", + "current_schema", + "current_user", + "equal_null", + "hll_sketch_estimate", + "hll_union", + "input_file_block_length", + "input_file_block_start", + "input_file_name", + "java_method", + "monotonically_increasing_id", + "reflect", + "spark_partition_id", + "try_aes_decrypt", + "typeof", + "user", + "uuid", + "version")) + .put( + GENERATOR, + Set.of( + "explode", + "explode_outer", + "inline", + "inline_outer", + "posexplode", + "posexplode_outer", + "stack")) + .build(); + + private static final Map FUNCTION_NAME_TO_FUNCTION_TYPE_MAP = + FUNCTION_TYPE_TO_FUNCTION_NAMES_MAP.entrySet().stream() + .flatMap( + entry -> entry.getValue().stream().map(value -> Map.entry(value, entry.getKey()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + public static FunctionType fromFunctionName(String functionName) { + return FUNCTION_NAME_TO_FUNCTION_TYPE_MAP.getOrDefault(functionName.toLowerCase(), UDF); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java new file mode 100644 index 0000000000..217640bada --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +enum GrammarElement { + ALTER_NAMESPACE("ALTER (DATABASE|TABLE|NAMESPACE)"), + ALTER_VIEW("ALTER VIEW"), + CREATE_NAMESPACE("CREATE (DATABASE|TABLE|NAMESPACE)"), + CREATE_FUNCTION("CREATE FUNCTION"), + CREATE_VIEW("CREATE VIEW"), + DROP_NAMESPACE("DROP (DATABASE|TABLE|NAMESPACE)"), + DROP_FUNCTION("DROP FUNCTION"), + DROP_VIEW("DROP VIEW"), + DROP_TABLE("DROP TABLE"), + REPAIR_TABLE("REPAIR TABLE"), + TRUNCATE_TABLE("TRUNCATE TABLE"), + // DML Statements + INSERT("INSERT"), + LOAD("LOAD"), + + // Data Retrieval Statements + EXPLAIN("EXPLAIN"), + WITH("WITH"), + CLUSTER_BY("CLUSTER BY"), + DISTRIBUTE_BY("DISTRIBUTE BY"), + // GROUP_BY("GROUP BY"), + // HAVING("HAVING"), + HINTS("HINTS"), + INLINE_TABLE("Inline Table(VALUES)"), + FILE("File"), + INNER_JOIN("INNER JOIN"), + CROSS_JOIN("CROSS JOIN"), + LEFT_OUTER_JOIN("LEFT OUTER JOIN"), + LEFT_SEMI_JOIN("LEFT SEMI JOIN"), + RIGHT_OUTER_JOIN("RIGHT OUTER JOIN"), + FULL_OUTER_JOIN("FULL OUTER JOIN"), + LEFT_ANTI_JOIN("LEFT ANTI JOIN"), + TABLESAMPLE("TABLESAMPLE"), + TABLE_VALUED_FUNCTION("Table-valued function"), + LATERAL_VIEW("LATERAL VIEW"), + LATERAL_SUBQUERY("LATERAL SUBQUERY"), + TRANSFORM("TRANSFORM"), + + // Auxiliary Statements + MANAGE_RESOURCE("Resource management statements"), + ANALYZE_TABLE("ANALYZE TABLE(S)"), + CACHE_TABLE("CACHE TABLE"), + CLEAR_CACHE("CLEAR CACHE"), + DESCRIBE_NAMESPACE("DESCRIBE (NAMESPACE|DATABASE|SCHEMA)"), + DESCRIBE_FUNCTION("DESCRIBE FUNCTION"), + DESCRIBE_QUERY("DESCRIBE QUERY"), + DESCRIBE_TABLE("DESCRIBE TABLE"), + REFRESH_RESOURCE("REFRESH"), + REFRESH_TABLE("REFRESH TABLE"), + REFRESH_FUNCTION("REFRESH FUNCTION"), + RESET("RESET"), + SET("SET"), + SHOW_COLUMNS("SHOW COLUMNS"), + SHOW_CREATE_TABLE("SHOW CREATE TABLE"), + SHOW_NAMESPACES("SHOW (DATABASES|SCHEMAS)"), + SHOW_FUNCTIONS("SHOW FUNCTIONS"), + SHOW_PARTITIONS("SHOW PARTITIONS"), + SHOW_TABLE_EXTENDED("SHOW TABLE EXTENDED"), + SHOW_TABLES("SHOW TABLES"), + SHOW_TBLPROPERTIES("SHOW TBLPROPERTIES"), + SHOW_VIEWS("SHOW VIEWS"), + UNCACHE_TABLE("UNCACHE TABLE"), + + // Functions + MAP_FUNCTIONS("Map functions"), + CSV_FUNCTIONS("CSV functions"), + MISC_FUNCTIONS("Misc functions"), + + // UDF + UDF("User Defined functions"); + + String description; + + @Override + public String toString() { + return description; + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidator.java new file mode 100644 index 0000000000..cc49643772 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidator.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +/** Interface for validator to decide if each GrammarElement is valid or not. */ +public interface GrammarElementValidator { + + /** + * @return true if element is valid (accepted) + */ + boolean isValid(GrammarElement element); +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProvider.java new file mode 100644 index 0000000000..9755a1c0b6 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProvider.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import java.util.Map; +import lombok.AllArgsConstructor; +import org.opensearch.sql.datasource.model.DataSourceType; + +/** Provides GrammarElementValidator based on DataSourceType. */ +@AllArgsConstructor +public class GrammarElementValidatorProvider { + + private final Map validatorMap; + private final GrammarElementValidator defaultValidator; + + public GrammarElementValidator getValidatorForDatasource(DataSourceType dataSourceType) { + return validatorMap.getOrDefault(dataSourceType, defaultValidator); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java new file mode 100644 index 0000000000..e7a0ce1b36 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.DISTRIBUTE_BY; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; +import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; +import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; +import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_RESOURCE; +import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.SET; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_VIEWS; +import static org.opensearch.sql.spark.validator.GrammarElement.TABLESAMPLE; +import static org.opensearch.sql.spark.validator.GrammarElement.TABLE_VALUED_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.TRANSFORM; +import static org.opensearch.sql.spark.validator.GrammarElement.UDF; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; + +public class S3GlueGrammarElementValidator extends DenyListGrammarElementValidator { + private static final Set S3GLUE_DENY_LIST = + ImmutableSet.builder() + .add( + ALTER_VIEW, + CREATE_FUNCTION, + CREATE_VIEW, + DROP_FUNCTION, + DROP_VIEW, + INSERT, + LOAD, + CLUSTER_BY, + DISTRIBUTE_BY, + HINTS, + INLINE_TABLE, + FILE, + TABLESAMPLE, + TABLE_VALUED_FUNCTION, + TRANSFORM, + MANAGE_RESOURCE, + DESCRIBE_FUNCTION, + REFRESH_RESOURCE, + REFRESH_FUNCTION, + RESET, + SET, + SHOW_FUNCTIONS, + SHOW_VIEWS, + MISC_FUNCTIONS, + UDF) + .build(); + + public S3GlueGrammarElementValidator() { + super(S3GLUE_DENY_LIST); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java new file mode 100644 index 0000000000..9ec0fb0109 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java @@ -0,0 +1,609 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTableColumnsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTablePartitionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterClusterByContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterTableAlterColumnContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewQueryContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewSchemaBindingContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeTablesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CacheTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClearCacheContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClusterBySpecContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateNamespaceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableLikeContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateViewContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CtesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DescribeFunctionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DescribeNamespaceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DescribeQueryContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DescribeRelationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropFunctionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropNamespaceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropTableColumnsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropTablePartitionsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropViewContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ExplainContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.FunctionNameContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.HintContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.HiveReplaceColumnsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InlineTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertIntoReplaceWhereContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertIntoTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertOverwriteDirContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertOverwriteHiveDirContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertOverwriteTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.JoinRelationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.JoinTypeContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.LateralViewContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.LoadDataContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ManageResourceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.QueryOrganizationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RecoverPartitionsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RefreshFunctionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RefreshResourceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RefreshTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RelationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RenameTableColumnContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RenameTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RenameTablePartitionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RepairTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ReplaceTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ResetConfigurationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ResetQuotedConfigurationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SampleContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetConfigurationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetNamespaceLocationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetNamespacePropertiesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetTableLocationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetTableSerDeContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowColumnsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowCreateTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowFunctionsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowNamespacesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowPartitionsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowTableExtendedContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowTablesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowTblPropertiesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowViewsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TableNameContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TableValuedFunctionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TransformClauseContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TruncateTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UncacheTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UnsetNamespacePropertiesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParserBaseVisitor; + +/** This visitor validate grammar using GrammarElementValidator */ +@AllArgsConstructor +public class SQLQueryValidationVisitor extends SqlBaseParserBaseVisitor { + private final GrammarElementValidator grammarElementValidator; + + @Override + public Void visitCreateFunction(SqlBaseParser.CreateFunctionContext ctx) { + validateAllowed(GrammarElement.CREATE_FUNCTION); + return super.visitCreateFunction(ctx); + } + + @Override + public Void visitSetNamespaceProperties(SetNamespacePropertiesContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitSetNamespaceProperties(ctx); + } + + @Override + public Void visitUnsetNamespaceProperties(UnsetNamespacePropertiesContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitUnsetNamespaceProperties(ctx); + } + + @Override + public Void visitAddTableColumns(AddTableColumnsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitAddTableColumns(ctx); + } + + @Override + public Void visitAddTablePartition(AddTablePartitionContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitAddTablePartition(ctx); + } + + @Override + public Void visitRenameTableColumn(RenameTableColumnContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitRenameTableColumn(ctx); + } + + @Override + public Void visitDropTableColumns(DropTableColumnsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitDropTableColumns(ctx); + } + + @Override + public Void visitAlterTableAlterColumn(AlterTableAlterColumnContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitAlterTableAlterColumn(ctx); + } + + @Override + public Void visitHiveReplaceColumns(HiveReplaceColumnsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitHiveReplaceColumns(ctx); + } + + @Override + public Void visitSetTableSerDe(SetTableSerDeContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitSetTableSerDe(ctx); + } + + @Override + public Void visitRenameTablePartition(RenameTablePartitionContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitRenameTablePartition(ctx); + } + + @Override + public Void visitDropTablePartitions(DropTablePartitionsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitDropTablePartitions(ctx); + } + + @Override + public Void visitSetTableLocation(SetTableLocationContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitSetTableLocation(ctx); + } + + @Override + public Void visitRecoverPartitions(RecoverPartitionsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitRecoverPartitions(ctx); + } + + @Override + public Void visitAlterClusterBy(AlterClusterByContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitAlterClusterBy(ctx); + } + + @Override + public Void visitSetNamespaceLocation(SetNamespaceLocationContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitSetNamespaceLocation(ctx); + } + + @Override + public Void visitAlterViewQuery(AlterViewQueryContext ctx) { + validateAllowed(GrammarElement.ALTER_VIEW); + return super.visitAlterViewQuery(ctx); + } + + @Override + public Void visitAlterViewSchemaBinding(AlterViewSchemaBindingContext ctx) { + validateAllowed(GrammarElement.ALTER_VIEW); + return super.visitAlterViewSchemaBinding(ctx); + } + + @Override + public Void visitRenameTable(RenameTableContext ctx) { + if (ctx.VIEW() != null) { + validateAllowed(GrammarElement.ALTER_VIEW); + } else { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + } + + return super.visitRenameTable(ctx); + } + + @Override + public Void visitCreateNamespace(CreateNamespaceContext ctx) { + validateAllowed(GrammarElement.CREATE_NAMESPACE); + return super.visitCreateNamespace(ctx); + } + + @Override + public Void visitCreateTable(CreateTableContext ctx) { + validateAllowed(GrammarElement.CREATE_NAMESPACE); + return super.visitCreateTable(ctx); + } + + @Override + public Void visitCreateTableLike(CreateTableLikeContext ctx) { + validateAllowed(GrammarElement.CREATE_NAMESPACE); + return super.visitCreateTableLike(ctx); + } + + @Override + public Void visitReplaceTable(ReplaceTableContext ctx) { + validateAllowed(GrammarElement.CREATE_NAMESPACE); + return super.visitReplaceTable(ctx); + } + + @Override + public Void visitDropNamespace(DropNamespaceContext ctx) { + validateAllowed(GrammarElement.DROP_NAMESPACE); + return super.visitDropNamespace(ctx); + } + + @Override + public Void visitDropTable(DropTableContext ctx) { + validateAllowed(GrammarElement.DROP_NAMESPACE); + return super.visitDropTable(ctx); + } + + @Override + public Void visitCreateView(CreateViewContext ctx) { + validateAllowed(GrammarElement.CREATE_VIEW); + return super.visitCreateView(ctx); + } + + @Override + public Void visitDropView(DropViewContext ctx) { + validateAllowed(GrammarElement.DROP_VIEW); + return super.visitDropView(ctx); + } + + @Override + public Void visitDropFunction(DropFunctionContext ctx) { + validateAllowed(GrammarElement.DROP_FUNCTION); + return super.visitDropFunction(ctx); + } + + @Override + public Void visitRepairTable(RepairTableContext ctx) { + validateAllowed(GrammarElement.REPAIR_TABLE); + return super.visitRepairTable(ctx); + } + + @Override + public Void visitTruncateTable(TruncateTableContext ctx) { + validateAllowed(GrammarElement.TRUNCATE_TABLE); + return super.visitTruncateTable(ctx); + } + + @Override + public Void visitInsertOverwriteTable(InsertOverwriteTableContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertOverwriteTable(ctx); + } + + @Override + public Void visitInsertIntoReplaceWhere(InsertIntoReplaceWhereContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertIntoReplaceWhere(ctx); + } + + @Override + public Void visitInsertIntoTable(InsertIntoTableContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertIntoTable(ctx); + } + + @Override + public Void visitInsertOverwriteDir(InsertOverwriteDirContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertOverwriteDir(ctx); + } + + @Override + public Void visitInsertOverwriteHiveDir(InsertOverwriteHiveDirContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertOverwriteHiveDir(ctx); + } + + @Override + public Void visitLoadData(LoadDataContext ctx) { + validateAllowed(GrammarElement.LOAD); + return super.visitLoadData(ctx); + } + + @Override + public Void visitExplain(ExplainContext ctx) { + validateAllowed(GrammarElement.EXPLAIN); + return super.visitExplain(ctx); + } + + @Override + public Void visitTableName(TableNameContext ctx) { + String reference = ctx.identifierReference().getText(); + if (isFileReference(reference)) { + validateAllowed(GrammarElement.FILE); + } + return super.visitTableName(ctx); + } + + private static final String FILE_REFERENCE_PATTERN = "^[a-zA-Z]+\\.`[^`]+`$"; + + private boolean isFileReference(String reference) { + return reference.matches(FILE_REFERENCE_PATTERN); + } + + @Override + public Void visitCtes(CtesContext ctx) { + validateAllowed(GrammarElement.WITH); + return super.visitCtes(ctx); + } + + @Override + public Void visitClusterBySpec(ClusterBySpecContext ctx) { + validateAllowed(GrammarElement.CLUSTER_BY); + return super.visitClusterBySpec(ctx); + } + + @Override + public Void visitQueryOrganization(QueryOrganizationContext ctx) { + if (ctx.CLUSTER() != null) { + validateAllowed(GrammarElement.CLUSTER_BY); + } else if (ctx.DISTRIBUTE() != null) { + validateAllowed(GrammarElement.DISTRIBUTE_BY); + } + return super.visitQueryOrganization(ctx); + } + + @Override + public Void visitHint(HintContext ctx) { + validateAllowed(GrammarElement.HINTS); + return super.visitHint(ctx); + } + + @Override + public Void visitInlineTable(InlineTableContext ctx) { + validateAllowed(GrammarElement.INLINE_TABLE); + return super.visitInlineTable(ctx); + } + + @Override + public Void visitJoinType(JoinTypeContext ctx) { + if (ctx.CROSS() != null) { + validateAllowed(GrammarElement.CROSS_JOIN); + } else if (ctx.LEFT() != null && ctx.SEMI() != null) { + validateAllowed(GrammarElement.LEFT_SEMI_JOIN); + } else if (ctx.ANTI() != null) { + validateAllowed(GrammarElement.LEFT_ANTI_JOIN); + } else if (ctx.LEFT() != null) { + validateAllowed(GrammarElement.LEFT_OUTER_JOIN); + } else if (ctx.RIGHT() != null) { + validateAllowed(GrammarElement.RIGHT_OUTER_JOIN); + } else if (ctx.FULL() != null) { + validateAllowed(GrammarElement.FULL_OUTER_JOIN); + } else { + validateAllowed(GrammarElement.INNER_JOIN); + } + return super.visitJoinType(ctx); + } + + @Override + public Void visitSample(SampleContext ctx) { + validateAllowed(GrammarElement.TABLESAMPLE); + return super.visitSample(ctx); + } + + @Override + public Void visitTableValuedFunction(TableValuedFunctionContext ctx) { + validateAllowed(GrammarElement.TABLE_VALUED_FUNCTION); + return super.visitTableValuedFunction(ctx); + } + + @Override + public Void visitLateralView(LateralViewContext ctx) { + validateAllowed(GrammarElement.LATERAL_VIEW); + return super.visitLateralView(ctx); + } + + @Override + public Void visitRelation(RelationContext ctx) { + if (ctx.LATERAL() != null) { + validateAllowed(GrammarElement.LATERAL_SUBQUERY); + } + return super.visitRelation(ctx); + } + + @Override + public Void visitJoinRelation(JoinRelationContext ctx) { + if (ctx.LATERAL() != null) { + validateAllowed(GrammarElement.LATERAL_SUBQUERY); + } + return super.visitJoinRelation(ctx); + } + + @Override + public Void visitTransformClause(TransformClauseContext ctx) { + if (ctx.TRANSFORM() != null) { + validateAllowed(GrammarElement.TRANSFORM); + } + return super.visitTransformClause(ctx); + } + + @Override + public Void visitManageResource(ManageResourceContext ctx) { + validateAllowed(GrammarElement.MANAGE_RESOURCE); + return super.visitManageResource(ctx); + } + + @Override + public Void visitAnalyze(AnalyzeContext ctx) { + validateAllowed(GrammarElement.ANALYZE_TABLE); + return super.visitAnalyze(ctx); + } + + @Override + public Void visitAnalyzeTables(AnalyzeTablesContext ctx) { + validateAllowed(GrammarElement.ANALYZE_TABLE); + return super.visitAnalyzeTables(ctx); + } + + @Override + public Void visitCacheTable(CacheTableContext ctx) { + validateAllowed(GrammarElement.CACHE_TABLE); + return super.visitCacheTable(ctx); + } + + @Override + public Void visitClearCache(ClearCacheContext ctx) { + validateAllowed(GrammarElement.CLEAR_CACHE); + return super.visitClearCache(ctx); + } + + @Override + public Void visitDescribeNamespace(DescribeNamespaceContext ctx) { + validateAllowed(GrammarElement.DESCRIBE_NAMESPACE); + return super.visitDescribeNamespace(ctx); + } + + @Override + public Void visitDescribeFunction(DescribeFunctionContext ctx) { + validateAllowed(GrammarElement.DESCRIBE_FUNCTION); + return super.visitDescribeFunction(ctx); + } + + @Override + public Void visitDescribeRelation(DescribeRelationContext ctx) { + validateAllowed(GrammarElement.DESCRIBE_TABLE); + return super.visitDescribeRelation(ctx); + } + + @Override + public Void visitDescribeQuery(DescribeQueryContext ctx) { + validateAllowed(GrammarElement.DESCRIBE_QUERY); + return super.visitDescribeQuery(ctx); + } + + @Override + public Void visitRefreshResource(RefreshResourceContext ctx) { + validateAllowed(GrammarElement.REFRESH_RESOURCE); + return super.visitRefreshResource(ctx); + } + + @Override + public Void visitRefreshTable(RefreshTableContext ctx) { + validateAllowed(GrammarElement.REFRESH_TABLE); + return super.visitRefreshTable(ctx); + } + + @Override + public Void visitRefreshFunction(RefreshFunctionContext ctx) { + validateAllowed(GrammarElement.REFRESH_FUNCTION); + return super.visitRefreshFunction(ctx); + } + + @Override + public Void visitResetConfiguration(ResetConfigurationContext ctx) { + validateAllowed(GrammarElement.RESET); + return super.visitResetConfiguration(ctx); + } + + @Override + public Void visitResetQuotedConfiguration(ResetQuotedConfigurationContext ctx) { + validateAllowed(GrammarElement.RESET); + return super.visitResetQuotedConfiguration(ctx); + } + + @Override + public Void visitSetConfiguration(SetConfigurationContext ctx) { + validateAllowed(GrammarElement.SET); + return super.visitSetConfiguration(ctx); + } + + @Override + public Void visitShowColumns(ShowColumnsContext ctx) { + validateAllowed(GrammarElement.SHOW_COLUMNS); + return super.visitShowColumns(ctx); + } + + @Override + public Void visitShowCreateTable(ShowCreateTableContext ctx) { + validateAllowed(GrammarElement.SHOW_CREATE_TABLE); + return super.visitShowCreateTable(ctx); + } + + @Override + public Void visitShowNamespaces(ShowNamespacesContext ctx) { + validateAllowed(GrammarElement.SHOW_NAMESPACES); + return super.visitShowNamespaces(ctx); + } + + @Override + public Void visitShowFunctions(ShowFunctionsContext ctx) { + validateAllowed(GrammarElement.SHOW_FUNCTIONS); + return super.visitShowFunctions(ctx); + } + + @Override + public Void visitShowPartitions(ShowPartitionsContext ctx) { + validateAllowed(GrammarElement.SHOW_PARTITIONS); + return super.visitShowPartitions(ctx); + } + + @Override + public Void visitShowTableExtended(ShowTableExtendedContext ctx) { + validateAllowed(GrammarElement.SHOW_TABLE_EXTENDED); + return super.visitShowTableExtended(ctx); + } + + @Override + public Void visitShowTables(ShowTablesContext ctx) { + validateAllowed(GrammarElement.SHOW_TABLES); + return super.visitShowTables(ctx); + } + + @Override + public Void visitShowTblProperties(ShowTblPropertiesContext ctx) { + validateAllowed(GrammarElement.SHOW_TBLPROPERTIES); + return super.visitShowTblProperties(ctx); + } + + @Override + public Void visitShowViews(ShowViewsContext ctx) { + validateAllowed(GrammarElement.SHOW_VIEWS); + return super.visitShowViews(ctx); + } + + @Override + public Void visitUncacheTable(UncacheTableContext ctx) { + validateAllowed(GrammarElement.UNCACHE_TABLE); + return super.visitUncacheTable(ctx); + } + + @Override + public Void visitFunctionName(FunctionNameContext ctx) { + validateFunctionAllowed(ctx.qualifiedName().getText()); + return super.visitFunctionName(ctx); + } + + private void validateFunctionAllowed(String function) { + FunctionType type = FunctionType.fromFunctionName(function.toLowerCase()); + switch (type) { + case MAP: + validateAllowed(GrammarElement.MAP_FUNCTIONS); + break; + case CSV: + validateAllowed(GrammarElement.CSV_FUNCTIONS); + break; + case MISC: + validateAllowed(GrammarElement.MISC_FUNCTIONS); + break; + case UDF: + validateAllowed(GrammarElement.UDF); + break; + } + } + + private void validateAllowed(GrammarElement element) { + if (!grammarElementValidator.isValid(element)) { + throw new IllegalArgumentException(element + " is not allowed."); + } + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java new file mode 100644 index 0000000000..f387cbad25 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.spark.utils.SQLQueryUtils; + +/** Validate input SQL query based on the DataSourceType. */ +@AllArgsConstructor +public class SQLQueryValidator { + private static final Logger log = LogManager.getLogger(SQLQueryValidator.class); + + private final GrammarElementValidatorProvider grammarElementValidatorProvider; + + /** + * It will look up validator associated with the DataSourceType, and throw + * IllegalArgumentException if invalid grammar element is found. + * + * @param sqlQuery The query to be validated + * @param datasourceType + */ + public void validate(String sqlQuery, DataSourceType datasourceType) { + GrammarElementValidator grammarElementValidator = + grammarElementValidatorProvider.getValidatorForDatasource(datasourceType); + SQLQueryValidationVisitor visitor = new SQLQueryValidationVisitor(grammarElementValidator); + try { + visitor.visit(SQLQueryUtils.getBaseParser(sqlQuery).singleStatement()); + } catch (IllegalArgumentException e) { + log.error("Query validation failed. DataSourceType=" + datasourceType, e); + throw e; + } + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java new file mode 100644 index 0000000000..ca8f2b5bdd --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java @@ -0,0 +1,113 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_NAMESPACE; +import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.ANALYZE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.CACHE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.CLEAR_CACHE; +import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_NAMESPACE; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CSV_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_NAMESPACE; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_QUERY; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.DISTRIBUTE_BY; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_NAMESPACE; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; +import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; +import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; +import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_RESOURCE; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.REPAIR_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.SET; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_COLUMNS; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_CREATE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_NAMESPACES; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_PARTITIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_TABLES; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_TABLE_EXTENDED; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_TBLPROPERTIES; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_VIEWS; +import static org.opensearch.sql.spark.validator.GrammarElement.TABLESAMPLE; +import static org.opensearch.sql.spark.validator.GrammarElement.TABLE_VALUED_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.TRANSFORM; +import static org.opensearch.sql.spark.validator.GrammarElement.TRUNCATE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.UDF; +import static org.opensearch.sql.spark.validator.GrammarElement.UNCACHE_TABLE; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; + +public class SecurityLakeGrammarElementValidator extends DenyListGrammarElementValidator { + private static final Set SECURITY_LAKE_DENY_LIST = + ImmutableSet.builder() + .add( + ALTER_NAMESPACE, + ALTER_VIEW, + CREATE_NAMESPACE, + CREATE_FUNCTION, + CREATE_VIEW, + DROP_FUNCTION, + DROP_NAMESPACE, + DROP_VIEW, + REPAIR_TABLE, + TRUNCATE_TABLE, + INSERT, + LOAD, + CLUSTER_BY, + DISTRIBUTE_BY, + HINTS, + INLINE_TABLE, + FILE, + TABLESAMPLE, + TABLE_VALUED_FUNCTION, + TRANSFORM, + MANAGE_RESOURCE, + ANALYZE_TABLE, + CACHE_TABLE, + CLEAR_CACHE, + DESCRIBE_NAMESPACE, + DESCRIBE_FUNCTION, + DESCRIBE_QUERY, + DESCRIBE_TABLE, + REFRESH_RESOURCE, + REFRESH_TABLE, + REFRESH_FUNCTION, + RESET, + SET, + SHOW_COLUMNS, + SHOW_CREATE_TABLE, + SHOW_NAMESPACES, + SHOW_FUNCTIONS, + SHOW_PARTITIONS, + SHOW_TABLE_EXTENDED, + SHOW_TABLES, + SHOW_TBLPROPERTIES, + SHOW_VIEWS, + UNCACHE_TABLE, + CSV_FUNCTIONS, + MISC_FUNCTIONS, + UDF) + .build(); + + public SecurityLakeGrammarElementValidator() { + super(SECURITY_LAKE_DENY_LIST); + } +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index ddadeb65e2..57ad4ecf42 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -85,6 +85,10 @@ import org.opensearch.sql.spark.rest.model.CreateAsyncQueryResponse; import org.opensearch.sql.spark.rest.model.LangType; import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; +import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; +import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; +import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.SQLQueryValidator; /** * This tests async-query-core library end-to-end using mocked implementation of extension points. @@ -175,9 +179,18 @@ public void setUp() { emrServerlessClientFactory, metricsService, new SparkSubmitParametersBuilderProvider(collection)); + SQLQueryValidator sqlQueryValidator = + new SQLQueryValidator( + new GrammarElementValidatorProvider( + ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueGrammarElementValidator()), + new DefaultGrammarElementValidator())); SparkQueryDispatcher sparkQueryDispatcher = new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, + sessionManager, + queryHandlerFactory, + queryIdProvider, + sqlQueryValidator); asyncQueryExecutorService = new AsyncQueryExecutorServiceImpl( asyncQueryJobMetadataStorageService, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 75c0e00337..1a38b6977f 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -42,6 +42,7 @@ import com.amazonaws.services.emrserverless.model.GetJobRunResult; import com.amazonaws.services.emrserverless.model.JobRun; import com.amazonaws.services.emrserverless.model.JobRunState; +import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -88,6 +89,10 @@ import org.opensearch.sql.spark.response.JobExecutionResponseReader; import org.opensearch.sql.spark.rest.model.LangType; import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; +import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; +import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; +import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.SQLQueryValidator; @ExtendWith(MockitoExtension.class) public class SparkQueryDispatcherTest { @@ -111,6 +116,13 @@ public class SparkQueryDispatcherTest { @Mock private AsyncQueryRequestContext asyncQueryRequestContext; @Mock private MetricsService metricsService; @Mock private AsyncQueryScheduler asyncQueryScheduler; + + private final SQLQueryValidator sqlQueryValidator = + new SQLQueryValidator( + new GrammarElementValidatorProvider( + ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueGrammarElementValidator()), + new DefaultGrammarElementValidator())); + private DataSourceSparkParameterComposer dataSourceSparkParameterComposer = (datasourceMetadata, sparkSubmitParameters, dispatchQueryRequest, context) -> { sparkSubmitParameters.setConfigItem(FLINT_INDEX_STORE_AUTH_KEY, "basic"); @@ -159,7 +171,11 @@ void setUp() { sparkSubmitParametersBuilderProvider); sparkQueryDispatcher = new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, + sessionManager, + queryHandlerFactory, + queryIdProvider, + sqlQueryValidator); } @Test @@ -347,19 +363,12 @@ void testDispatchWithSparkUDFQuery() { sparkQueryDispatcher.dispatch( getBaseDispatchQueryRequestBuilder(query).langType(LangType.SQL).build(), asyncQueryRequestContext)); - assertEquals( - "Query is not allowed: Creating user-defined functions is not allowed", - illegalArgumentException.getMessage()); + assertEquals("CREATE FUNCTION is not allowed.", illegalArgumentException.getMessage()); verifyNoInteractions(emrServerlessClient); verifyNoInteractions(flintIndexMetadataService); } } - @Test - void testInvalidSQLQueryDispatchToSpark() { - testDispatchBatchQuery("myselect 1"); - } - @Test void testDispatchQueryWithoutATableAndDataSourceName() { testDispatchBatchQuery("show tables"); @@ -571,7 +580,11 @@ void testDispatchAlterToManualRefreshIndexQuery() { QueryHandlerFactory queryHandlerFactory = mock(QueryHandlerFactory.class); sparkQueryDispatcher = new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, + sessionManager, + queryHandlerFactory, + queryIdProvider, + sqlQueryValidator); String query = "ALTER INDEX elb_and_requestUri ON my_glue.default.http_logs WITH" + " (auto_refresh = false)"; @@ -597,7 +610,11 @@ void testDispatchDropIndexQuery() { QueryHandlerFactory queryHandlerFactory = mock(QueryHandlerFactory.class); sparkQueryDispatcher = new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, + sessionManager, + queryHandlerFactory, + queryIdProvider, + sqlQueryValidator); String query = "DROP INDEX elb_and_requestUri ON my_glue.default.http_logs"; DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java index 56cab7ce7f..881ad0e56a 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java @@ -10,7 +10,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; import static org.opensearch.sql.spark.utils.SQLQueryUtilsTest.IndexQuery.index; import static org.opensearch.sql.spark.utils.SQLQueryUtilsTest.IndexQuery.mv; import static org.opensearch.sql.spark.utils.SQLQueryUtilsTest.IndexQuery.skippingIndex; @@ -22,7 +21,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.datasource.model.DataSource; -import org.opensearch.sql.datasource.model.DataSourceType; import org.opensearch.sql.spark.dispatcher.model.FullyQualifiedTableName; import org.opensearch.sql.spark.dispatcher.model.IndexQueryActionType; import org.opensearch.sql.spark.dispatcher.model.IndexQueryDetails; @@ -444,106 +442,6 @@ void testRecoverIndex() { assertEquals(IndexQueryActionType.RECOVER, indexDetails.getIndexQueryActionType()); } - @Test - void testValidateSparkSqlQuery_ValidQuery() { - List errors = - validateSparkSqlQueryForDataSourceType( - "DELETE FROM Customers WHERE CustomerName='Alfreds Futterkiste'", - DataSourceType.PROMETHEUS); - - assertTrue(errors.isEmpty(), "Valid query should not produce any errors"); - } - - @Test - void testValidateSparkSqlQuery_SelectQuery_DataSourceSecurityLake() { - List errors = - validateSparkSqlQueryForDataSourceType( - "SELECT * FROM users WHERE age > 18", DataSourceType.SECURITY_LAKE); - - assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); - } - - @Test - void testValidateSparkSqlQuery_SelectQuery_DataSourceTypeNull() { - List errors = - validateSparkSqlQueryForDataSourceType("SELECT * FROM users WHERE age > 18", null); - - assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); - } - - @Test - void testValidateSparkSqlQuery_InvalidQuery_SyntaxCheckFailureSkippedWithoutValidationError() { - List errors = - validateSparkSqlQueryForDataSourceType( - "SEECT * FROM users WHERE age > 18", DataSourceType.SECURITY_LAKE); - - assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); - } - - @Test - void testValidateSparkSqlQuery_nullDatasource() { - List errors = - SQLQueryUtils.validateSparkSqlQuery(null, "SELECT * FROM users WHERE age > 18"); - assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); - } - - private List validateSparkSqlQueryForDataSourceType( - String query, DataSourceType dataSourceType) { - when(this.dataSource.getConnectorType()).thenReturn(dataSourceType); - - return SQLQueryUtils.validateSparkSqlQuery(this.dataSource, query); - } - - @Test - void testValidateSparkSqlQuery_SelectQuery_DataSourceSecurityLake_ValidationFails() { - List errors = - validateSparkSqlQueryForDataSourceType( - "REFRESH INDEX cv1 ON mys3.default.http_logs", DataSourceType.SECURITY_LAKE); - - assertFalse( - errors.isEmpty(), - "Invalid query as Security Lake datasource supports only flint queries and SELECT sql" - + " queries. Given query was REFRESH sql query"); - assertEquals( - errors.get(0), - "Unsupported sql statement for security lake data source. Only select queries are allowed"); - } - - @Test - void - testValidateSparkSqlQuery_NonSelectStatementContainingSelectClause_DataSourceSecurityLake_ValidationFails() { - String query = - "CREATE TABLE AccountSummaryOrWhatever AS " - + "select taxid, address1, count(address1) from dbo.t " - + "group by taxid, address1;"; - - List errors = - validateSparkSqlQueryForDataSourceType(query, DataSourceType.SECURITY_LAKE); - - assertFalse( - errors.isEmpty(), - "Invalid query as Security Lake datasource supports only flint queries and SELECT sql" - + " queries. Given query was REFRESH sql query"); - assertEquals( - errors.get(0), - "Unsupported sql statement for security lake data source. Only select queries are allowed"); - } - - @Test - void testValidateSparkSqlQuery_InvalidQuery() { - when(dataSource.getConnectorType()).thenReturn(DataSourceType.PROMETHEUS); - String invalidQuery = "CREATE FUNCTION myUDF AS 'com.example.UDF'"; - - List errors = SQLQueryUtils.validateSparkSqlQuery(dataSource, invalidQuery); - - assertFalse(errors.isEmpty(), "Invalid query should produce errors"); - assertEquals(1, errors.size(), "Should have one error"); - assertEquals( - "Creating user-defined functions is not allowed", - errors.get(0), - "Error message should match"); - } - @Getter protected static class IndexQuery { private String query; diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/FunctionTypeTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/FunctionTypeTest.java new file mode 100644 index 0000000000..a5f868421c --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/FunctionTypeTest.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class FunctionTypeTest { + @Test + public void test() { + assertEquals(FunctionType.AGGREGATE, FunctionType.fromFunctionName("any")); + assertEquals(FunctionType.AGGREGATE, FunctionType.fromFunctionName("variance")); + assertEquals(FunctionType.WINDOW, FunctionType.fromFunctionName("cume_dist")); + assertEquals(FunctionType.WINDOW, FunctionType.fromFunctionName("row_number")); + assertEquals(FunctionType.ARRAY, FunctionType.fromFunctionName("array")); + assertEquals(FunctionType.ARRAY, FunctionType.fromFunctionName("sort_array")); + assertEquals(FunctionType.MAP, FunctionType.fromFunctionName("element_at")); + assertEquals(FunctionType.MAP, FunctionType.fromFunctionName("try_element_at")); + assertEquals(FunctionType.DATE_TIMESTAMP, FunctionType.fromFunctionName("add_months")); + assertEquals(FunctionType.DATE_TIMESTAMP, FunctionType.fromFunctionName("year")); + assertEquals(FunctionType.JSON, FunctionType.fromFunctionName("from_json")); + assertEquals(FunctionType.JSON, FunctionType.fromFunctionName("to_json")); + assertEquals(FunctionType.MATH, FunctionType.fromFunctionName("abs")); + assertEquals(FunctionType.MATH, FunctionType.fromFunctionName("width_bucket")); + assertEquals(FunctionType.STRING, FunctionType.fromFunctionName("ascii")); + assertEquals(FunctionType.STRING, FunctionType.fromFunctionName("upper")); + assertEquals(FunctionType.CONDITIONAL, FunctionType.fromFunctionName("coalesce")); + assertEquals(FunctionType.CONDITIONAL, FunctionType.fromFunctionName("nvl2")); + assertEquals(FunctionType.BITWISE, FunctionType.fromFunctionName("bit_count")); + assertEquals(FunctionType.BITWISE, FunctionType.fromFunctionName("shiftrightunsigned")); + assertEquals(FunctionType.CONVERSION, FunctionType.fromFunctionName("bigint")); + assertEquals(FunctionType.CONVERSION, FunctionType.fromFunctionName("tinyint")); + assertEquals(FunctionType.PREDICATE, FunctionType.fromFunctionName("isnan")); + assertEquals(FunctionType.PREDICATE, FunctionType.fromFunctionName("rlike")); + assertEquals(FunctionType.CSV, FunctionType.fromFunctionName("from_csv")); + assertEquals(FunctionType.CSV, FunctionType.fromFunctionName("to_csv")); + assertEquals(FunctionType.MISC, FunctionType.fromFunctionName("aes_decrypt")); + assertEquals(FunctionType.MISC, FunctionType.fromFunctionName("version")); + assertEquals(FunctionType.GENERATOR, FunctionType.fromFunctionName("explode")); + assertEquals(FunctionType.GENERATOR, FunctionType.fromFunctionName("stack")); + assertEquals(FunctionType.UDF, FunctionType.fromFunctionName("unknown")); + } +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java new file mode 100644 index 0000000000..7d4b255356 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.datasource.model.DataSourceType; + +class GrammarElementValidatorProviderTest { + S3GlueGrammarElementValidator s3GlueGrammarElementValidator = new S3GlueGrammarElementValidator(); + SecurityLakeGrammarElementValidator securityLakeGrammarElementValidator = + new SecurityLakeGrammarElementValidator(); + DefaultGrammarElementValidator defaultGrammarElementValidator = + new DefaultGrammarElementValidator(); + GrammarElementValidatorProvider grammarElementValidatorProvider = + new GrammarElementValidatorProvider( + ImmutableMap.of( + DataSourceType.S3GLUE, s3GlueGrammarElementValidator, + DataSourceType.SECURITY_LAKE, securityLakeGrammarElementValidator), + defaultGrammarElementValidator); + + @Test + public void test() { + assertEquals( + s3GlueGrammarElementValidator, + grammarElementValidatorProvider.getValidatorForDatasource(DataSourceType.S3GLUE)); + assertEquals( + securityLakeGrammarElementValidator, + grammarElementValidatorProvider.getValidatorForDatasource(DataSourceType.SECURITY_LAKE)); + assertEquals( + defaultGrammarElementValidator, + grammarElementValidatorProvider.getValidatorForDatasource(DataSourceType.PROMETHEUS)); + } +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java new file mode 100644 index 0000000000..6726b56994 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -0,0 +1,600 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.antlr.v4.runtime.CommonTokenStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.spark.antlr.parser.SqlBaseLexer; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SingleStatementContext; + +@ExtendWith(MockitoExtension.class) +class SQLQueryValidatorTest { + @Mock GrammarElementValidatorProvider mockedProvider; + + @InjectMocks SQLQueryValidator sqlQueryValidator; + + private enum TestElement { + // DDL Statements + ALTER_DATABASE( + "ALTER DATABASE inventory SET DBPROPERTIES ('Edit-date' = '01/01/2001');", + "ALTER DATABASE dbx.tab1 UNSET PROPERTIES ('winner');", + "ALTER DATABASE dbx.tab1 SET LOCATION '/path/to/part/ways';"), + ALTER_TABLE( + "ALTER TABLE default.StudentInfo PARTITION (age='10') RENAME TO PARTITION (age='15');", + "ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner');", + "ALTER TABLE StudentInfo ADD columns (LastName string, DOB timestamp);", + "ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18);", + "ALTER TABLE StudentInfo RENAME COLUMN name TO FirstName;", + "ALTER TABLE StudentInfo RENAME TO newName;", + "ALTER TABLE StudentInfo DROP columns (LastName, DOB);", + "ALTER TABLE StudentInfo ALTER COLUMN FirstName COMMENT \"new comment\";", + "ALTER TABLE StudentInfo REPLACE COLUMNS (name string, ID int COMMENT 'new comment');", + "ALTER TABLE test_tab SET SERDE 'org.apache.LazyBinaryColumnarSerDe';", + "ALTER TABLE StudentInfo DROP IF EXISTS PARTITION (age=18);", + "ALTER TABLE dbx.tab1 PARTITION (a='1', b='2') SET LOCATION '/path/to/part/ways';", + "ALTER TABLE dbx.tab1 RECOVER PARTITIONS;", + "ALTER TABLE dbx.tab1 CLUSTER BY NONE;", + "ALTER TABLE dbx.tab1 SET LOCATION '/path/to/part/ways';"), + ALTER_VIEW( + "ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;", + "ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;", + "ALTER VIEW tempdb1.v2 WITH SCHEMA BINDING"), + CREATE_DATABASE("CREATE DATABASE IF NOT EXISTS customer_db;\n"), + CREATE_FUNCTION("CREATE FUNCTION simple_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar';"), + CREATE_TABLE( + "CREATE TABLE Student_Dupli like Student;", + "CREATE TABLE student (id INT, name STRING, age INT) USING CSV;", + "CREATE TABLE student_copy USING CSV AS SELECT * FROM student;", + "CREATE TABLE student (id INT, name STRING, age INT);", + "REPLACE TABLE student (id INT, name STRING, age INT) USING CSV;"), + CREATE_VIEW( + "CREATE OR REPLACE VIEW experienced_employee" + + " (ID COMMENT 'Unique identification number', Name)" + + " COMMENT 'View for experienced employees'" + + " AS SELECT id, name FROM all_employee" + + " WHERE working_years > 5;"), + DROP_DATABASE("DROP DATABASE inventory_db CASCADE;"), + DROP_FUNCTION("DROP FUNCTION test_avg;"), + DROP_TABLE("DROP TABLE employeetable;"), + DROP_VIEW("DROP VIEW employeeView;"), + REPAIR_TABLE("REPAIR TABLE t1;"), + TRUNCATE_TABLE("TRUNCATE TABLE Student partition(age=10);"), + + // DML Statements + INSERT_TABLE( + "INSERT INTO target_table SELECT * FROM source_table;", + "INSERT INTO persons REPLACE WHERE ssn = 123456789 SELECT * FROM persons2;", + "INSERT OVERWRITE students VALUES ('Ashua Hill', '456 Erica Ct, Cupertino', 111111);"), + INSERT_OVERWRITE_DIRECTORY( + "INSERT OVERWRITE DIRECTORY '/path/to/output' SELECT * FROM source_table;", + "INSERT OVERWRITE DIRECTORY USING myTable SELECT * FROM source_table;", + "INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination' STORED AS orc SELECT * FROM" + + " test_table;"), + LOAD("LOAD DATA INPATH '/path/to/data' INTO TABLE target_table;"), + + // Data Retrieval Statements + SELECT("SELECT 1;"), + EXPLAIN("EXPLAIN SELECT * FROM my_table;"), + COMMON_TABLE_EXPRESSION( + "WITH cte AS (SELECT * FROM my_table WHERE age > 30) SELECT * FROM cte;"), + CLUSTER_BY_CLAUSE( + "SELECT * FROM my_table CLUSTER BY age;", "ALTER TABLE testTable CLUSTER BY (age);"), + DISTRIBUTE_BY_CLAUSE("SELECT * FROM my_table DISTRIBUTE BY name;"), + GROUP_BY_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name;"), + HAVING_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name HAVING count(*) > 1;"), + HINTS("SELECT /*+ BROADCAST(my_table) */ * FROM my_table;"), + INLINE_TABLE("SELECT * FROM (VALUES (1, 'a'), (2, 'b')) AS inline_table(id, value);"), + FILE("SELECT * FROM text.`/path/to/file.txt`;"), + INNER_JOIN("SELECT t1.name, t2.age FROM table1 t1 INNER JOIN table2 t2 ON t1.id = t2.id;"), + CROSS_JOIN("SELECT t1.name, t2.age FROM table1 t1 CROSS JOIN table2 t2;"), + LEFT_OUTER_JOIN( + "SELECT t1.name, t2.age FROM table1 t1 LEFT OUTER JOIN table2 t2 ON t1.id = t2.id;"), + LEFT_SEMI_JOIN("SELECT t1.name FROM table1 t1 LEFT SEMI JOIN table2 t2 ON t1.id = t2.id;"), + RIGHT_OUTER_JOIN( + "SELECT t1.name, t2.age FROM table1 t1 RIGHT OUTER JOIN table2 t2 ON t1.id = t2.id;"), + FULL_OUTER_JOIN( + "SELECT t1.name, t2.age FROM table1 t1 FULL OUTER JOIN table2 t2 ON t1.id = t2.id;"), + LEFT_ANTI_JOIN("SELECT t1.name FROM table1 t1 LEFT ANTI JOIN table2 t2 ON t1.id = t2.id;"), + LIKE_PREDICATE("SELECT * FROM my_table WHERE name LIKE 'A%';"), + LIMIT_CLAUSE("SELECT * FROM my_table LIMIT 10;"), + OFFSET_CLAUSE("SELECT * FROM my_table OFFSET 5;"), + ORDER_BY_CLAUSE("SELECT * FROM my_table ORDER BY age DESC;"), + SET_OPERATORS("SELECT * FROM table1 UNION SELECT * FROM table2;"), + SORT_BY_CLAUSE("SELECT * FROM my_table SORT BY age DESC;"), + TABLESAMPLE("SELECT * FROM my_table TABLESAMPLE(10 PERCENT);"), + // TABLE_VALUED_FUNCTION("SELECT explode(array(10, 20));"), TODO: Need to handle this case + TABLE_VALUED_FUNCTION("SELECT * FROM explode(array(10, 20));"), + WHERE_CLAUSE("SELECT * FROM my_table WHERE age > 30;"), + AGGREGATE_FUNCTION("SELECT count(*) FROM my_table;"), + WINDOW_FUNCTION("SELECT name, age, rank() OVER (ORDER BY age DESC) FROM my_table;"), + CASE_CLAUSE("SELECT name, CASE WHEN age > 30 THEN 'Adult' ELSE 'Young' END FROM my_table;"), + PIVOT_CLAUSE( + "SELECT * FROM (SELECT name, age, gender FROM my_table) PIVOT (COUNT(*) FOR gender IN ('M'," + + " 'F'));"), + UNPIVOT_CLAUSE( + "SELECT name, value, category FROM (SELECT name, 'M' AS gender, age AS male_age, 0 AS" + + " female_age FROM my_table) UNPIVOT (value FOR category IN (male_age, female_age));"), + LATERAL_VIEW_CLAUSE( + "SELECT name, age, exploded_value FROM my_table LATERAL VIEW OUTER EXPLODE(split(comments," + + " ',')) exploded_table AS exploded_value;"), + LATERAL_SUBQUERY( + "SELECT * FROM t1, LATERAL (SELECT * FROM t2 WHERE t1.c1 = t2.c1);", + "SELECT * FROM t1 JOIN LATERAL (SELECT * FROM t2 WHERE t1.c1 = t2.c1);"), + TRANSFORM_CLAUSE( + "SELECT transform(zip_code, name, age) USING 'cat' AS (a, b, c) FROM my_table;"), + + // Auxiliary Statements + ADD_FILE("ADD FILE /tmp/test.txt;"), + ADD_JAR("ADD JAR /path/to/my.jar;"), + ANALYZE_TABLE( + "ANALYZE TABLE my_table COMPUTE STATISTICS;", + "ANALYZE TABLES IN school_db COMPUTE STATISTICS NOSCAN;"), + CACHE_TABLE("CACHE TABLE my_table;"), + CLEAR_CACHE("CLEAR CACHE;"), + DESCRIBE_DATABASE("DESCRIBE DATABASE my_db;"), + DESCRIBE_FUNCTION("DESCRIBE FUNCTION my_function;"), + DESCRIBE_QUERY("DESCRIBE QUERY SELECT * FROM my_table;"), + DESCRIBE_TABLE("DESCRIBE TABLE my_table;"), + LIST_FILE("LIST FILE '/path/to/files';"), + LIST_JAR("LIST JAR;"), + REFRESH("REFRESH;"), + REFRESH_TABLE("REFRESH TABLE my_table;"), + REFRESH_FUNCTION("REFRESH FUNCTION my_function;"), + RESET("RESET;", "RESET spark.abc;", "RESET `key`;"), + SET( + "SET spark.sql.shuffle.partitions=200;", + "SET -v;", + "SET;", + "SET spark.sql.variable.substitute;"), + SHOW_COLUMNS("SHOW COLUMNS FROM my_table;"), + SHOW_CREATE_TABLE("SHOW CREATE TABLE my_table;"), + SHOW_DATABASES("SHOW DATABASES;"), + SHOW_FUNCTIONS("SHOW FUNCTIONS;"), + SHOW_PARTITIONS("SHOW PARTITIONS my_table;"), + SHOW_TABLE_EXTENDED("SHOW TABLE EXTENDED LIKE 'my_table';"), + SHOW_TABLES("SHOW TABLES;"), + SHOW_TBLPROPERTIES("SHOW TBLPROPERTIES my_table;"), + SHOW_VIEWS("SHOW VIEWS;"), + UNCACHE_TABLE("UNCACHE TABLE my_table;"), + + // Functions + ARRAY_FUNCTIONS("SELECT array_contains(array(1, 2, 3), 2);"), + MAP_FUNCTIONS("SELECT map_keys(map('a', 1, 'b', 2));"), + DATE_AND_TIMESTAMP_FUNCTIONS("SELECT date_format(current_date(), 'yyyy-MM-dd');"), + JSON_FUNCTIONS("SELECT json_tuple('{\"a\":1, \"b\":2}', 'a', 'b');"), + MATHEMATICAL_FUNCTIONS("SELECT round(3.1415, 2);"), + STRING_FUNCTIONS("SELECT ascii('Hello');"), + BITWISE_FUNCTIONS("SELECT bit_count(42);"), + CONVERSION_FUNCTIONS("SELECT cast('2023-04-01' as date);"), + CONDITIONAL_FUNCTIONS("SELECT if(1 > 0, 'true', 'false');"), + PREDICATE_FUNCTIONS("SELECT isnotnull(1);"), + CSV_FUNCTIONS("SELECT from_csv(array('a', 'b', 'c'), ',');"), + MISC_FUNCTIONS("SELECT current_user();"), + + // Aggregate-like Functions + AGGREGATE_FUNCTIONS("SELECT count(*), max(age), min(age) FROM my_table;"), + WINDOW_FUNCTIONS("SELECT name, age, rank() OVER (ORDER BY age DESC) FROM my_table;"), + + // Generator Functions + GENERATOR_FUNCTIONS("SELECT explode(array(1, 2, 3));"), + + // UDFs (User-Defined Functions) + SCALAR_USER_DEFINED_FUNCTIONS("SELECT my_udf(name) FROM my_table;"), + USER_DEFINED_AGGREGATE_FUNCTIONS("SELECT my_udaf(age) FROM my_table GROUP BY name;"), + INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS("SELECT my_hive_udf(name) FROM my_table;"); + + @Getter private final String[] queries; + + TestElement(String... queries) { + this.queries = queries; + } + } + + @Test + void testAllowAllByDefault() { + when(mockedProvider.getValidatorForDatasource(any())) + .thenReturn(new DefaultGrammarElementValidator()); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SPARK); + Arrays.stream(TestElement.values()).forEach(v::ok); + } + + @Test + void testDenyAllValidator() { + when(mockedProvider.getValidatorForDatasource(any())).thenReturn(element -> false); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SPARK); + // The elements which doesn't have validation will be accepted. + // That's why there are some ok case + + // DDL Statements + v.ng(TestElement.ALTER_DATABASE); + v.ng(TestElement.ALTER_TABLE); + v.ng(TestElement.ALTER_VIEW); + v.ng(TestElement.CREATE_DATABASE); + v.ng(TestElement.CREATE_FUNCTION); + v.ng(TestElement.CREATE_TABLE); + v.ng(TestElement.CREATE_VIEW); + v.ng(TestElement.DROP_DATABASE); + v.ng(TestElement.DROP_FUNCTION); + v.ng(TestElement.DROP_TABLE); + v.ng(TestElement.DROP_VIEW); + v.ng(TestElement.REPAIR_TABLE); + v.ng(TestElement.TRUNCATE_TABLE); + + // DML Statements + v.ng(TestElement.INSERT_TABLE); + v.ng(TestElement.INSERT_OVERWRITE_DIRECTORY); + v.ng(TestElement.LOAD); + + // Data Retrieval + v.ng(TestElement.EXPLAIN); + v.ng(TestElement.COMMON_TABLE_EXPRESSION); + v.ng(TestElement.CLUSTER_BY_CLAUSE); + v.ng(TestElement.DISTRIBUTE_BY_CLAUSE); + v.ok(TestElement.GROUP_BY_CLAUSE); + v.ok(TestElement.HAVING_CLAUSE); + v.ng(TestElement.HINTS); + v.ng(TestElement.INLINE_TABLE); + v.ng(TestElement.FILE); + v.ng(TestElement.INNER_JOIN); + v.ng(TestElement.CROSS_JOIN); + v.ng(TestElement.LEFT_OUTER_JOIN); + v.ng(TestElement.LEFT_SEMI_JOIN); + v.ng(TestElement.RIGHT_OUTER_JOIN); + v.ng(TestElement.FULL_OUTER_JOIN); + v.ng(TestElement.LEFT_ANTI_JOIN); + v.ok(TestElement.LIKE_PREDICATE); + v.ok(TestElement.LIMIT_CLAUSE); + v.ok(TestElement.OFFSET_CLAUSE); + v.ok(TestElement.ORDER_BY_CLAUSE); + v.ok(TestElement.SET_OPERATORS); + v.ok(TestElement.SORT_BY_CLAUSE); + v.ng(TestElement.TABLESAMPLE); + v.ng(TestElement.TABLE_VALUED_FUNCTION); + v.ok(TestElement.WHERE_CLAUSE); + v.ok(TestElement.AGGREGATE_FUNCTION); + v.ok(TestElement.WINDOW_FUNCTION); + v.ok(TestElement.CASE_CLAUSE); + v.ok(TestElement.PIVOT_CLAUSE); + v.ok(TestElement.UNPIVOT_CLAUSE); + v.ng(TestElement.LATERAL_VIEW_CLAUSE); + v.ng(TestElement.LATERAL_SUBQUERY); + v.ng(TestElement.TRANSFORM_CLAUSE); + + // Auxiliary Statements + v.ng(TestElement.ADD_FILE); + v.ng(TestElement.ADD_JAR); + v.ng(TestElement.ANALYZE_TABLE); + v.ng(TestElement.CACHE_TABLE); + v.ng(TestElement.CLEAR_CACHE); + v.ng(TestElement.DESCRIBE_DATABASE); + v.ng(TestElement.DESCRIBE_FUNCTION); + v.ng(TestElement.DESCRIBE_QUERY); + v.ng(TestElement.DESCRIBE_TABLE); + v.ng(TestElement.LIST_FILE); + v.ng(TestElement.LIST_JAR); + v.ng(TestElement.REFRESH); + v.ng(TestElement.REFRESH_TABLE); + v.ng(TestElement.REFRESH_FUNCTION); + v.ng(TestElement.RESET); + v.ng(TestElement.SET); + v.ng(TestElement.SHOW_COLUMNS); + v.ng(TestElement.SHOW_CREATE_TABLE); + v.ng(TestElement.SHOW_DATABASES); + v.ng(TestElement.SHOW_FUNCTIONS); + v.ng(TestElement.SHOW_PARTITIONS); + v.ng(TestElement.SHOW_TABLE_EXTENDED); + v.ng(TestElement.SHOW_TABLES); + v.ng(TestElement.SHOW_TBLPROPERTIES); + v.ng(TestElement.SHOW_VIEWS); + v.ng(TestElement.UNCACHE_TABLE); + + // Functions + v.ok(TestElement.ARRAY_FUNCTIONS); + v.ng(TestElement.MAP_FUNCTIONS); + v.ok(TestElement.DATE_AND_TIMESTAMP_FUNCTIONS); + v.ok(TestElement.JSON_FUNCTIONS); + v.ok(TestElement.MATHEMATICAL_FUNCTIONS); + v.ok(TestElement.STRING_FUNCTIONS); + v.ok(TestElement.BITWISE_FUNCTIONS); + v.ok(TestElement.CONVERSION_FUNCTIONS); + v.ok(TestElement.CONDITIONAL_FUNCTIONS); + v.ok(TestElement.PREDICATE_FUNCTIONS); + v.ng(TestElement.CSV_FUNCTIONS); + v.ng(TestElement.MISC_FUNCTIONS); + + // Aggregate-like Functions + v.ok(TestElement.AGGREGATE_FUNCTIONS); + v.ok(TestElement.WINDOW_FUNCTIONS); + + // Generator Functions + v.ok(TestElement.GENERATOR_FUNCTIONS); + + // UDFs + v.ng(TestElement.SCALAR_USER_DEFINED_FUNCTIONS); + v.ng(TestElement.USER_DEFINED_AGGREGATE_FUNCTIONS); + v.ng(TestElement.INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS); + } + + @Test + void testS3glueQueries() { + when(mockedProvider.getValidatorForDatasource(any())) + .thenReturn(new S3GlueGrammarElementValidator()); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.S3GLUE); + + // DDL Statements + v.ok(TestElement.ALTER_DATABASE); + v.ok(TestElement.ALTER_TABLE); + v.ng(TestElement.ALTER_VIEW); + v.ok(TestElement.CREATE_DATABASE); + v.ng(TestElement.CREATE_FUNCTION); + v.ok(TestElement.CREATE_TABLE); + v.ng(TestElement.CREATE_VIEW); + v.ok(TestElement.DROP_DATABASE); + v.ng(TestElement.DROP_FUNCTION); + v.ok(TestElement.DROP_TABLE); + v.ng(TestElement.DROP_VIEW); + v.ok(TestElement.REPAIR_TABLE); + v.ok(TestElement.TRUNCATE_TABLE); + + // DML Statements + v.ng(TestElement.INSERT_TABLE); + v.ng(TestElement.INSERT_OVERWRITE_DIRECTORY); + v.ng(TestElement.LOAD); + + // Data Retrieval + v.ok(TestElement.SELECT); + v.ok(TestElement.EXPLAIN); + v.ok(TestElement.COMMON_TABLE_EXPRESSION); + v.ng(TestElement.CLUSTER_BY_CLAUSE); + v.ng(TestElement.DISTRIBUTE_BY_CLAUSE); + v.ok(TestElement.GROUP_BY_CLAUSE); + v.ok(TestElement.HAVING_CLAUSE); + v.ng(TestElement.HINTS); + v.ng(TestElement.INLINE_TABLE); + v.ng(TestElement.FILE); + v.ok(TestElement.INNER_JOIN); + v.ok(TestElement.CROSS_JOIN); + v.ok(TestElement.LEFT_OUTER_JOIN); + v.ok(TestElement.LEFT_SEMI_JOIN); + v.ok(TestElement.RIGHT_OUTER_JOIN); + v.ok(TestElement.FULL_OUTER_JOIN); + v.ok(TestElement.LEFT_ANTI_JOIN); + v.ok(TestElement.LIKE_PREDICATE); + v.ok(TestElement.LIMIT_CLAUSE); + v.ok(TestElement.OFFSET_CLAUSE); + v.ok(TestElement.ORDER_BY_CLAUSE); + v.ok(TestElement.SET_OPERATORS); + v.ok(TestElement.SORT_BY_CLAUSE); + v.ng(TestElement.TABLESAMPLE); + v.ng(TestElement.TABLE_VALUED_FUNCTION); + v.ok(TestElement.WHERE_CLAUSE); + v.ok(TestElement.AGGREGATE_FUNCTION); + v.ok(TestElement.WINDOW_FUNCTION); + v.ok(TestElement.CASE_CLAUSE); + v.ok(TestElement.PIVOT_CLAUSE); + v.ok(TestElement.UNPIVOT_CLAUSE); + v.ok(TestElement.LATERAL_VIEW_CLAUSE); + v.ok(TestElement.LATERAL_SUBQUERY); + v.ng(TestElement.TRANSFORM_CLAUSE); + + // Auxiliary Statements + v.ng(TestElement.ADD_FILE); + v.ng(TestElement.ADD_JAR); + v.ok(TestElement.ANALYZE_TABLE); + v.ok(TestElement.CACHE_TABLE); + v.ok(TestElement.CLEAR_CACHE); + v.ok(TestElement.DESCRIBE_DATABASE); + v.ng(TestElement.DESCRIBE_FUNCTION); + v.ok(TestElement.DESCRIBE_QUERY); + v.ok(TestElement.DESCRIBE_TABLE); + v.ng(TestElement.LIST_FILE); + v.ng(TestElement.LIST_JAR); + v.ng(TestElement.REFRESH); + v.ok(TestElement.REFRESH_TABLE); + v.ng(TestElement.REFRESH_FUNCTION); + v.ng(TestElement.RESET); + v.ng(TestElement.SET); + v.ok(TestElement.SHOW_COLUMNS); + v.ok(TestElement.SHOW_CREATE_TABLE); + v.ok(TestElement.SHOW_DATABASES); + v.ng(TestElement.SHOW_FUNCTIONS); + v.ok(TestElement.SHOW_PARTITIONS); + v.ok(TestElement.SHOW_TABLE_EXTENDED); + v.ok(TestElement.SHOW_TABLES); + v.ok(TestElement.SHOW_TBLPROPERTIES); + v.ng(TestElement.SHOW_VIEWS); + v.ok(TestElement.UNCACHE_TABLE); + + // Functions + v.ok(TestElement.ARRAY_FUNCTIONS); + v.ok(TestElement.MAP_FUNCTIONS); + v.ok(TestElement.DATE_AND_TIMESTAMP_FUNCTIONS); + v.ok(TestElement.JSON_FUNCTIONS); + v.ok(TestElement.MATHEMATICAL_FUNCTIONS); + v.ok(TestElement.STRING_FUNCTIONS); + v.ok(TestElement.BITWISE_FUNCTIONS); + v.ok(TestElement.CONVERSION_FUNCTIONS); + v.ok(TestElement.CONDITIONAL_FUNCTIONS); + v.ok(TestElement.PREDICATE_FUNCTIONS); + v.ok(TestElement.CSV_FUNCTIONS); + v.ng(TestElement.MISC_FUNCTIONS); + + // Aggregate-like Functions + v.ok(TestElement.AGGREGATE_FUNCTIONS); + v.ok(TestElement.WINDOW_FUNCTIONS); + + // Generator Functions + v.ok(TestElement.GENERATOR_FUNCTIONS); + + // UDFs + v.ng(TestElement.SCALAR_USER_DEFINED_FUNCTIONS); + v.ng(TestElement.USER_DEFINED_AGGREGATE_FUNCTIONS); + v.ng(TestElement.INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS); + } + + @Test + void testSecurityLakeQueries() { + when(mockedProvider.getValidatorForDatasource(any())) + .thenReturn(new SecurityLakeGrammarElementValidator()); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SECURITY_LAKE); + + // DDL Statements + v.ng(TestElement.ALTER_DATABASE); + v.ng(TestElement.ALTER_TABLE); + v.ng(TestElement.ALTER_VIEW); + v.ng(TestElement.CREATE_DATABASE); + v.ng(TestElement.CREATE_FUNCTION); + v.ng(TestElement.CREATE_TABLE); + v.ng(TestElement.CREATE_VIEW); + v.ng(TestElement.DROP_DATABASE); + v.ng(TestElement.DROP_FUNCTION); + v.ng(TestElement.DROP_TABLE); + v.ng(TestElement.DROP_VIEW); + v.ng(TestElement.REPAIR_TABLE); + v.ng(TestElement.TRUNCATE_TABLE); + + // DML Statements + v.ng(TestElement.INSERT_TABLE); + v.ng(TestElement.INSERT_OVERWRITE_DIRECTORY); + v.ng(TestElement.LOAD); + + // Data Retrieval + v.ok(TestElement.SELECT); + v.ok(TestElement.EXPLAIN); + v.ok(TestElement.COMMON_TABLE_EXPRESSION); + v.ng(TestElement.CLUSTER_BY_CLAUSE); + v.ng(TestElement.DISTRIBUTE_BY_CLAUSE); + v.ok(TestElement.GROUP_BY_CLAUSE); + v.ok(TestElement.HAVING_CLAUSE); + v.ng(TestElement.HINTS); + v.ng(TestElement.INLINE_TABLE); + v.ng(TestElement.FILE); + v.ok(TestElement.INNER_JOIN); + v.ok(TestElement.CROSS_JOIN); + v.ok(TestElement.LEFT_OUTER_JOIN); + v.ok(TestElement.LEFT_SEMI_JOIN); + v.ok(TestElement.RIGHT_OUTER_JOIN); + v.ok(TestElement.FULL_OUTER_JOIN); + v.ok(TestElement.LEFT_ANTI_JOIN); + v.ok(TestElement.LIKE_PREDICATE); + v.ok(TestElement.LIMIT_CLAUSE); + v.ok(TestElement.OFFSET_CLAUSE); + v.ok(TestElement.ORDER_BY_CLAUSE); + v.ok(TestElement.SET_OPERATORS); + v.ok(TestElement.SORT_BY_CLAUSE); + v.ng(TestElement.TABLESAMPLE); + v.ng(TestElement.TABLE_VALUED_FUNCTION); + v.ok(TestElement.WHERE_CLAUSE); + v.ok(TestElement.AGGREGATE_FUNCTION); + v.ok(TestElement.WINDOW_FUNCTION); + v.ok(TestElement.CASE_CLAUSE); + v.ok(TestElement.PIVOT_CLAUSE); + v.ok(TestElement.UNPIVOT_CLAUSE); + v.ok(TestElement.LATERAL_VIEW_CLAUSE); + v.ok(TestElement.LATERAL_SUBQUERY); + v.ng(TestElement.TRANSFORM_CLAUSE); + + // Auxiliary Statements + v.ng(TestElement.ADD_FILE); + v.ng(TestElement.ADD_JAR); + v.ng(TestElement.ANALYZE_TABLE); + v.ng(TestElement.CACHE_TABLE); + v.ng(TestElement.CLEAR_CACHE); + v.ng(TestElement.DESCRIBE_DATABASE); + v.ng(TestElement.DESCRIBE_FUNCTION); + v.ng(TestElement.DESCRIBE_QUERY); + v.ng(TestElement.DESCRIBE_TABLE); + v.ng(TestElement.LIST_FILE); + v.ng(TestElement.LIST_JAR); + v.ng(TestElement.REFRESH); + v.ng(TestElement.REFRESH_TABLE); + v.ng(TestElement.REFRESH_FUNCTION); + v.ng(TestElement.RESET); + v.ng(TestElement.SET); + v.ng(TestElement.SHOW_COLUMNS); + v.ng(TestElement.SHOW_CREATE_TABLE); + v.ng(TestElement.SHOW_DATABASES); + v.ng(TestElement.SHOW_FUNCTIONS); + v.ng(TestElement.SHOW_PARTITIONS); + v.ng(TestElement.SHOW_TABLE_EXTENDED); + v.ng(TestElement.SHOW_TABLES); + v.ng(TestElement.SHOW_TBLPROPERTIES); + v.ng(TestElement.SHOW_VIEWS); + v.ng(TestElement.UNCACHE_TABLE); + + // Functions + v.ok(TestElement.ARRAY_FUNCTIONS); + v.ok(TestElement.MAP_FUNCTIONS); + v.ok(TestElement.DATE_AND_TIMESTAMP_FUNCTIONS); + v.ok(TestElement.JSON_FUNCTIONS); + v.ok(TestElement.MATHEMATICAL_FUNCTIONS); + v.ok(TestElement.STRING_FUNCTIONS); + v.ok(TestElement.BITWISE_FUNCTIONS); + v.ok(TestElement.CONVERSION_FUNCTIONS); + v.ok(TestElement.CONDITIONAL_FUNCTIONS); + v.ok(TestElement.PREDICATE_FUNCTIONS); + v.ng(TestElement.CSV_FUNCTIONS); + v.ng(TestElement.MISC_FUNCTIONS); + + // Aggregate-like Functions + v.ok(TestElement.AGGREGATE_FUNCTIONS); + v.ok(TestElement.WINDOW_FUNCTIONS); + + // Generator Functions + v.ok(TestElement.GENERATOR_FUNCTIONS); + + // UDFs + v.ng(TestElement.SCALAR_USER_DEFINED_FUNCTIONS); + v.ng(TestElement.USER_DEFINED_AGGREGATE_FUNCTIONS); + v.ng(TestElement.INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS); + } + + @AllArgsConstructor + private static class VerifyValidator { + private final SQLQueryValidator validator; + private final DataSourceType dataSourceType; + + public void ok(TestElement query) { + runValidate(query.getQueries()); + } + + public void ng(TestElement query) { + assertThrows( + IllegalArgumentException.class, + () -> runValidate(query.getQueries()), + "The query should throw: query=`" + query.toString() + "`"); + } + + void runValidate(String[] queries) { + Arrays.stream(queries).forEach(query -> validator.validate(query, dataSourceType)); + } + + void runValidate(String query) { + validator.validate(query, dataSourceType); + } + + SingleStatementContext getParser(String query) { + SqlBaseParser sqlBaseParser = + new SqlBaseParser( + new CommonTokenStream(new SqlBaseLexer(new CaseInsensitiveCharStream(query)))); + return sqlBaseParser.singleStatement(); + } + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java index c6f6ffcd81..db070182a3 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.spark.execution.statestore.StateStore.ALL_DATASOURCE; +import com.google.common.collect.ImmutableMap; import lombok.RequiredArgsConstructor; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.service.ClusterService; @@ -64,6 +65,11 @@ import org.opensearch.sql.spark.response.OpenSearchJobExecutionResponseReader; import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; +import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; +import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; +import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.SQLQueryValidator; +import org.opensearch.sql.spark.validator.SecurityLakeGrammarElementValidator; @RequiredArgsConstructor public class AsyncExecutorServiceModule extends AbstractModule { @@ -101,9 +107,10 @@ public SparkQueryDispatcher sparkQueryDispatcher( DataSourceService dataSourceService, SessionManager sessionManager, QueryHandlerFactory queryHandlerFactory, - QueryIdProvider queryIdProvider) { + QueryIdProvider queryIdProvider, + SQLQueryValidator sqlQueryValidator) { return new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider, sqlQueryValidator); } @Provides @@ -174,6 +181,19 @@ public SparkSubmitParametersBuilderProvider sparkSubmitParametersBuilderProvider return new SparkSubmitParametersBuilderProvider(collection); } + @Provides + public SQLQueryValidator sqlQueryValidator() { + GrammarElementValidatorProvider validatorProvider = + new GrammarElementValidatorProvider( + ImmutableMap.of( + DataSourceType.S3GLUE, + new S3GlueGrammarElementValidator(), + DataSourceType.SECURITY_LAKE, + new SecurityLakeGrammarElementValidator()), + new DefaultGrammarElementValidator()); + return new SQLQueryValidator(validatorProvider); + } + @Provides public IndexDMLResultStorageService indexDMLResultStorageService( DataSourceService dataSourceService, StateStore stateStore) { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java index db0adfc156..175f9ac914 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java @@ -312,7 +312,7 @@ public void withSessionCreateAsyncQueryFailed() { // 1. create async query. CreateAsyncQueryResponse response = asyncQueryExecutorService.createAsyncQuery( - new CreateAsyncQueryRequest("myselect 1", MYS3_DATASOURCE, LangType.SQL, null), + new CreateAsyncQueryRequest("select 1", MYS3_DATASOURCE, LangType.SQL, null), asyncQueryRequestContext); assertNotNull(response.getSessionId()); Optional statementModel = diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java index 9b897d36b4..72ed17f5aa 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java @@ -102,6 +102,10 @@ import org.opensearch.sql.spark.response.OpenSearchJobExecutionResponseReader; import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; +import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; +import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; +import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.SQLQueryValidator; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.test.OpenSearchIntegTestCase; @@ -308,6 +312,11 @@ protected AsyncQueryExecutorService createAsyncQueryExecutorService( emrServerlessClientFactory, new OpenSearchMetricsService(), sparkSubmitParametersBuilderProvider); + SQLQueryValidator sqlQueryValidator = + new SQLQueryValidator( + new GrammarElementValidatorProvider( + ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueGrammarElementValidator()), + new DefaultGrammarElementValidator())); SparkQueryDispatcher sparkQueryDispatcher = new SparkQueryDispatcher( this.dataSourceService, @@ -318,7 +327,8 @@ protected AsyncQueryExecutorService createAsyncQueryExecutorService( sessionConfigSupplier, sessionIdProvider), queryHandlerFactory, - new DatasourceEmbeddedQueryIdProvider()); + new DatasourceEmbeddedQueryIdProvider(), + sqlQueryValidator); return new AsyncQueryExecutorServiceImpl( asyncQueryJobMetadataStorageService, sparkQueryDispatcher, From b71cb1c863bc2b8a2b05d34505977dc8a5272d16 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA Date: Fri, 27 Sep 2024 16:39:31 -0700 Subject: [PATCH 04/32] Remove unneeded Dummy files (#3018) Signed-off-by: Tomoyuki Morita --- .../org/opensearch/sql/asyncquery/Dummy.java | 13 --------- .../opensearch/sql/asyncquery/DummyTest.java | 18 ------------ .../sql/asyncquery/DummyConsumer.java | 18 ------------ .../sql/asyncquery/DummyConsumerTest.java | 28 ------------------- 4 files changed, 77 deletions(-) delete mode 100644 async-query-core/src/main/java/org/opensearch/sql/asyncquery/Dummy.java delete mode 100644 async-query-core/src/test/java/org/opensearch/sql/asyncquery/DummyTest.java delete mode 100644 async-query/src/main/java/org/opensearch/sql/asyncquery/DummyConsumer.java delete mode 100644 async-query/src/test/java/org/opensearch/sql/asyncquery/DummyConsumerTest.java diff --git a/async-query-core/src/main/java/org/opensearch/sql/asyncquery/Dummy.java b/async-query-core/src/main/java/org/opensearch/sql/asyncquery/Dummy.java deleted file mode 100644 index b7ab572f2a..0000000000 --- a/async-query-core/src/main/java/org/opensearch/sql/asyncquery/Dummy.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.asyncquery; - -// This is a dummy class for scaffolding and should be deleted later -public class Dummy { - public String hello() { - return "Hello!"; - } -} diff --git a/async-query-core/src/test/java/org/opensearch/sql/asyncquery/DummyTest.java b/async-query-core/src/test/java/org/opensearch/sql/asyncquery/DummyTest.java deleted file mode 100644 index 8fa1cf49ec..0000000000 --- a/async-query-core/src/test/java/org/opensearch/sql/asyncquery/DummyTest.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.asyncquery; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -public class DummyTest { - @Test - public void test() { - Dummy dummy = new Dummy(); - assertEquals("Hello!", dummy.hello()); - } -} diff --git a/async-query/src/main/java/org/opensearch/sql/asyncquery/DummyConsumer.java b/async-query/src/main/java/org/opensearch/sql/asyncquery/DummyConsumer.java deleted file mode 100644 index 9b1641e559..0000000000 --- a/async-query/src/main/java/org/opensearch/sql/asyncquery/DummyConsumer.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.asyncquery; - -import lombok.AllArgsConstructor; - -// This is a dummy class for scaffolding and should be deleted later -@AllArgsConstructor -public class DummyConsumer { - Dummy dummy; - - public String hello() { - return dummy.hello(); - } -} diff --git a/async-query/src/test/java/org/opensearch/sql/asyncquery/DummyConsumerTest.java b/async-query/src/test/java/org/opensearch/sql/asyncquery/DummyConsumerTest.java deleted file mode 100644 index a08dbae736..0000000000 --- a/async-query/src/test/java/org/opensearch/sql/asyncquery/DummyConsumerTest.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.asyncquery; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.when; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class DummyConsumerTest { - - @Mock Dummy dummy; - - @Test - public void test() { - DummyConsumer dummyConsumer = new DummyConsumer(dummy); - when(dummy.hello()).thenReturn("Hello from mock"); - - assertEquals("Hello from mock", dummyConsumer.hello()); - } -} From 724176cc97dc0adec1c4e9a0aa25472413ac68a4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Sun, 29 Sep 2024 15:34:24 +0800 Subject: [PATCH 05/32] Increment version to 2.18.0-SNAPSHOT (#2993) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 645a128101..afe18cec59 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.17.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.18.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') From 34a5fe50c2b7017ea7431084ede5f4f011f802cb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:05:58 -0700 Subject: [PATCH 06/32] Disable join types in validators (#3056) (#3060) * Disable join types in validators * Removed methods from SQLQueryValidationVisitor due to grammar file change --------- (cherry picked from commit ac8678c60c3e98c7e3791b2abb2b0e71c643256c) Signed-off-by: Tomoyuki Morita Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../S3GlueGrammarElementValidator.java | 10 +++++++ .../validator/SQLQueryValidationVisitor.java | 28 ------------------ .../SecurityLakeGrammarElementValidator.java | 10 +++++++ .../validator/SQLQueryValidatorTest.java | 29 ++++++++----------- 4 files changed, 32 insertions(+), 45 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java index e7a0ce1b36..9ed1fd9e9e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java @@ -9,20 +9,25 @@ import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CROSS_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.DISTRIBUTE_BY; import static org.opensearch.sql.spark.validator.GrammarElement.DROP_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.FULL_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_ANTI_JOIN; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_SEMI_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_RESOURCE; import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.RIGHT_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.SET; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_VIEWS; @@ -50,6 +55,11 @@ public class S3GlueGrammarElementValidator extends DenyListGrammarElementValidat HINTS, INLINE_TABLE, FILE, + CROSS_JOIN, + LEFT_SEMI_JOIN, + RIGHT_OUTER_JOIN, + FULL_OUTER_JOIN, + LEFT_ANTI_JOIN, TABLESAMPLE, TABLE_VALUED_FUNCTION, TRANSFORM, diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java index 9ec0fb0109..d50503418e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java @@ -9,15 +9,12 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTableColumnsContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTablePartitionContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterClusterByContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterTableAlterColumnContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewQueryContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewSchemaBindingContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeTablesContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CacheTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClearCacheContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClusterBySpecContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateNamespaceContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableLikeContext; @@ -81,7 +78,6 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TransformClauseContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TruncateTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UncacheTableContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UnsetNamespacePropertiesContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParserBaseVisitor; /** This visitor validate grammar using GrammarElementValidator */ @@ -101,12 +97,6 @@ public Void visitSetNamespaceProperties(SetNamespacePropertiesContext ctx) { return super.visitSetNamespaceProperties(ctx); } - @Override - public Void visitUnsetNamespaceProperties(UnsetNamespacePropertiesContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); - return super.visitUnsetNamespaceProperties(ctx); - } - @Override public Void visitAddTableColumns(AddTableColumnsContext ctx) { validateAllowed(GrammarElement.ALTER_NAMESPACE); @@ -173,12 +163,6 @@ public Void visitRecoverPartitions(RecoverPartitionsContext ctx) { return super.visitRecoverPartitions(ctx); } - @Override - public Void visitAlterClusterBy(AlterClusterByContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); - return super.visitAlterClusterBy(ctx); - } - @Override public Void visitSetNamespaceLocation(SetNamespaceLocationContext ctx) { validateAllowed(GrammarElement.ALTER_NAMESPACE); @@ -191,12 +175,6 @@ public Void visitAlterViewQuery(AlterViewQueryContext ctx) { return super.visitAlterViewQuery(ctx); } - @Override - public Void visitAlterViewSchemaBinding(AlterViewSchemaBindingContext ctx) { - validateAllowed(GrammarElement.ALTER_VIEW); - return super.visitAlterViewSchemaBinding(ctx); - } - @Override public Void visitRenameTable(RenameTableContext ctx) { if (ctx.VIEW() != null) { @@ -337,12 +315,6 @@ public Void visitCtes(CtesContext ctx) { return super.visitCtes(ctx); } - @Override - public Void visitClusterBySpec(ClusterBySpecContext ctx) { - validateAllowed(GrammarElement.CLUSTER_BY); - return super.visitClusterBySpec(ctx); - } - @Override public Void visitQueryOrganization(QueryOrganizationContext ctx) { if (ctx.CLUSTER() != null) { diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java index ca8f2b5bdd..7dd2b0ee89 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java @@ -14,6 +14,7 @@ import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_NAMESPACE; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CROSS_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.CSV_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_NAMESPACE; @@ -24,9 +25,12 @@ import static org.opensearch.sql.spark.validator.GrammarElement.DROP_NAMESPACE; import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.FULL_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_ANTI_JOIN; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_SEMI_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; @@ -35,6 +39,7 @@ import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.REPAIR_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.RIGHT_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.SET; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_COLUMNS; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_CREATE_TABLE; @@ -76,6 +81,11 @@ public class SecurityLakeGrammarElementValidator extends DenyListGrammarElementV HINTS, INLINE_TABLE, FILE, + CROSS_JOIN, + LEFT_SEMI_JOIN, + RIGHT_OUTER_JOIN, + FULL_OUTER_JOIN, + LEFT_ANTI_JOIN, TABLESAMPLE, TABLE_VALUED_FUNCTION, TRANSFORM, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java index 6726b56994..695a083809 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -34,11 +34,9 @@ private enum TestElement { // DDL Statements ALTER_DATABASE( "ALTER DATABASE inventory SET DBPROPERTIES ('Edit-date' = '01/01/2001');", - "ALTER DATABASE dbx.tab1 UNSET PROPERTIES ('winner');", "ALTER DATABASE dbx.tab1 SET LOCATION '/path/to/part/ways';"), ALTER_TABLE( "ALTER TABLE default.StudentInfo PARTITION (age='10') RENAME TO PARTITION (age='15');", - "ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner');", "ALTER TABLE StudentInfo ADD columns (LastName string, DOB timestamp);", "ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18);", "ALTER TABLE StudentInfo RENAME COLUMN name TO FirstName;", @@ -50,12 +48,10 @@ private enum TestElement { "ALTER TABLE StudentInfo DROP IF EXISTS PARTITION (age=18);", "ALTER TABLE dbx.tab1 PARTITION (a='1', b='2') SET LOCATION '/path/to/part/ways';", "ALTER TABLE dbx.tab1 RECOVER PARTITIONS;", - "ALTER TABLE dbx.tab1 CLUSTER BY NONE;", "ALTER TABLE dbx.tab1 SET LOCATION '/path/to/part/ways';"), ALTER_VIEW( "ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;", - "ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;", - "ALTER VIEW tempdb1.v2 WITH SCHEMA BINDING"), + "ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;"), CREATE_DATABASE("CREATE DATABASE IF NOT EXISTS customer_db;\n"), CREATE_FUNCTION("CREATE FUNCTION simple_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar';"), CREATE_TABLE( @@ -94,8 +90,7 @@ private enum TestElement { EXPLAIN("EXPLAIN SELECT * FROM my_table;"), COMMON_TABLE_EXPRESSION( "WITH cte AS (SELECT * FROM my_table WHERE age > 30) SELECT * FROM cte;"), - CLUSTER_BY_CLAUSE( - "SELECT * FROM my_table CLUSTER BY age;", "ALTER TABLE testTable CLUSTER BY (age);"), + CLUSTER_BY_CLAUSE("SELECT * FROM my_table CLUSTER BY age;"), DISTRIBUTE_BY_CLAUSE("SELECT * FROM my_table DISTRIBUTE BY name;"), GROUP_BY_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name;"), HAVING_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name HAVING count(*) > 1;"), @@ -370,12 +365,12 @@ void testS3glueQueries() { v.ng(TestElement.INLINE_TABLE); v.ng(TestElement.FILE); v.ok(TestElement.INNER_JOIN); - v.ok(TestElement.CROSS_JOIN); + v.ng(TestElement.CROSS_JOIN); v.ok(TestElement.LEFT_OUTER_JOIN); - v.ok(TestElement.LEFT_SEMI_JOIN); - v.ok(TestElement.RIGHT_OUTER_JOIN); - v.ok(TestElement.FULL_OUTER_JOIN); - v.ok(TestElement.LEFT_ANTI_JOIN); + v.ng(TestElement.LEFT_SEMI_JOIN); + v.ng(TestElement.RIGHT_OUTER_JOIN); + v.ng(TestElement.FULL_OUTER_JOIN); + v.ng(TestElement.LEFT_ANTI_JOIN); v.ok(TestElement.LIKE_PREDICATE); v.ok(TestElement.LIMIT_CLAUSE); v.ok(TestElement.OFFSET_CLAUSE); @@ -487,12 +482,12 @@ void testSecurityLakeQueries() { v.ng(TestElement.INLINE_TABLE); v.ng(TestElement.FILE); v.ok(TestElement.INNER_JOIN); - v.ok(TestElement.CROSS_JOIN); + v.ng(TestElement.CROSS_JOIN); v.ok(TestElement.LEFT_OUTER_JOIN); - v.ok(TestElement.LEFT_SEMI_JOIN); - v.ok(TestElement.RIGHT_OUTER_JOIN); - v.ok(TestElement.FULL_OUTER_JOIN); - v.ok(TestElement.LEFT_ANTI_JOIN); + v.ng(TestElement.LEFT_SEMI_JOIN); + v.ng(TestElement.RIGHT_OUTER_JOIN); + v.ng(TestElement.FULL_OUTER_JOIN); + v.ng(TestElement.LEFT_ANTI_JOIN); v.ok(TestElement.LIKE_PREDICATE); v.ok(TestElement.LIMIT_CLAUSE); v.ok(TestElement.OFFSET_CLAUSE); From 5f6f0b4fac73f7d08a8ee11ffd3d5e46de09ead2 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 08:12:40 -0700 Subject: [PATCH 07/32] CVE-2024-6345 fix (#2931) (#3065) (cherry picked from commit 063015c8290551090b9025687fa40078d435adfb) Signed-off-by: Heng Qian Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- doctest/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doctest/requirements.txt b/doctest/requirements.txt index 7d178b80ae..7d5e2afa2d 100644 --- a/doctest/requirements.txt +++ b/doctest/requirements.txt @@ -1 +1,2 @@ -zc.customdoctests==1.0.1 \ No newline at end of file +zc.customdoctests==1.0.1 +setuptools>=70.0.0 From ede56da35d19ff6d34c56540b84565af615c9090 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:28:44 -0700 Subject: [PATCH 08/32] [Backport 2.x] Merge feature/pit branch to main (#2939) * Add pit for join queries (#2703) * Add search after for join * Enable search after by default * Add pit * nit * Fix tests * ignore joinWithGeoIntersectNL * Rerun CI with scroll * Remove unused code and retrigger CI with search_after true * Address comments * Remove unused code change * Update pit keep alive time with SQL_CURSOR_KEEP_ALIVE * Fix scroll condition * nit * Add pit before query execution * nit * Move pit from join request builder to executor * Remove unused methods * Add pit in parent class's run() * Add comment for fetching subsequent result in NestedLoopsElasticExecutor * Update comment * Add javadoc for pit handler * Add pit interface * Add pit handler unit test * Fix failed unit test CI * Fix spotless error * Rename pit class and add logs * Fix pit delete unit test --------- * Add pit for multi query (#2753) * Add search after for join * Enable search after by default * Add pit * nit * Fix tests * ignore joinWithGeoIntersectNL * Rerun CI with scroll * draft * Remove unused code and retrigger CI with search_after true * Address comments * Remove unused code change * Update pit keep alive time with SQL_CURSOR_KEEP_ALIVE * Fix scroll condition * nit * Add pit before query execution * Refactor get response with pit method * Update remaining scroll search calls * Fix integ test failures * nit * Move pit from join request builder to executor * Remove unused methods * Move pit from request to executor * Fix pit.delete call missed while merge * Move getResponseWithHits method to util class * add try catch for create delete pit in minus executor * move all common fields to ElasticHitsExecutor * add javadoc for ElasticHitsExecutor * Add missing javadoc * Forcing an empty commit as last commit is stuck processing updates --------- --------- (cherry picked from commit 7815c96597585c6c47ed6f6189cd1c6c8b56f0a8) Signed-off-by: Rupal Mahajan Signed-off-by: Manasvini B S Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] Co-authored-by: Rupal Mahajan --- .../sql/common/setting/Settings.java | 1 + docs/dev/Pagination.md | 40 ++++ docs/user/admin/settings.rst | 44 +++++ .../org/opensearch/sql/legacy/JoinIT.java | 6 +- .../legacy/executor/ElasticHitsExecutor.java | 91 ++++++++- .../executor/join/ElasticJoinExecutor.java | 76 ++++---- .../join/HashJoinElasticExecutor.java | 39 ++-- .../join/NestedLoopsElasticExecutor.java | 33 ++-- .../join/QueryPlanElasticExecutor.java | 5 +- .../legacy/executor/multi/MinusExecutor.java | 181 ++++++++++-------- .../legacy/executor/multi/UnionExecutor.java | 3 +- .../sql/legacy/pit/PointInTimeHandler.java | 18 ++ .../legacy/pit/PointInTimeHandlerImpl.java | 83 ++++++++ .../pit/PointInTimeHandlerImplTest.java | 81 ++++++++ .../setting/OpenSearchSettings.java | 14 ++ 15 files changed, 557 insertions(+), 158 deletions(-) create mode 100644 legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandler.java create mode 100644 legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java create mode 100644 legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 32babad03f..0037032d22 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -23,6 +23,7 @@ public enum Key { SQL_SLOWLOG("plugins.sql.slowlog"), SQL_CURSOR_KEEP_ALIVE("plugins.sql.cursor.keep_alive"), SQL_DELETE_ENABLED("plugins.sql.delete.enabled"), + SQL_PAGINATION_API_SEARCH_AFTER("plugins.sql.pagination.api"), /** PPL Settings. */ PPL_ENABLED("plugins.ppl.enabled"), diff --git a/docs/dev/Pagination.md b/docs/dev/Pagination.md index 4982b13d7f..1919af30fe 100644 --- a/docs/dev/Pagination.md +++ b/docs/dev/Pagination.md @@ -477,4 +477,44 @@ Response: } +``` + +#### plugins.sql.pagination.api + +This setting controls whether the SQL search queries in OpenSearch use Point-In-Time (PIT) with search_after or the traditional scroll mechanism for fetching paginated results. + +- Default Value: true +- Possible Values: true or false +- When set to true, the search query in the background uses PIT with search_after instead of scroll to retrieve paginated results. The Cursor Id returned to the user will encode relevant pagination query-related information, which will be used to fetch the subsequent pages of results. +- This setting is node-level. +- This setting can be updated dynamically. + +Example: + +``` +>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings -d '{ + "transient" : { + "plugins.sql.pagination.api" : "true" + } +}' +``` + +Response: + +``` +{ + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "plugins" : { + "sql" : { + "pagination" : { + "api" : "true" + } + } + } + } +} + + ``` diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 0e644dc910..71718d1726 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -196,6 +196,50 @@ Result set:: Note: the legacy settings of ``opendistro.sql.cursor.keep_alive`` is deprecated, it will fallback to the new settings if you request an update with the legacy name. +plugins.sql.pagination.api +================================ + +Description +----------- + +This setting controls whether the SQL search queries in OpenSearch use Point-In-Time (PIT) with search_after or the traditional scroll mechanism for fetching paginated results. + +1. Default Value: true +2. Possible Values: true or false +3. When set to true, the search query in the background uses PIT with search_after instead of scroll to retrieve paginated results. The Cursor Id returned to the user will encode relevant pagination query-related information, which will be used to fetch the subsequent pages of results. +4. This setting is node-level. +5. This setting can be updated dynamically. + + +Example +------- + +You can update the setting with a new value like this. + +SQL query:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.sql.pagination.api" : "true" + } + }' + +Result set:: + + { + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "plugins" : { + "sql" : { + "pagination" : { + "api" : "true" + } + } + } + } + } + plugins.query.size_limit =========================== diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java index 8019454b77..8c2ea96474 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java @@ -288,6 +288,8 @@ public void hintMultiSearchCanRunFewTimesNL() throws IOException { Assert.assertThat(hits.length(), equalTo(42)); } + // TODO: Fix joinWithGeoIntersectNL test when SQL_PAGINATION_API_SEARCH_AFTER is true + @Ignore @Test public void joinWithGeoIntersectNL() throws IOException { @@ -455,7 +457,7 @@ public void joinParseCheckSelectedFieldsSplitNLConditionOrderGT() throws IOExcep "SELECT /*! USE_NL*/ a.firstname, a.lastname, a.gender, d.firstname, d.age FROM %s a" + " JOIN %s d on a.age < d.age WHERE (d.firstname = 'Lynn' OR d.firstname =" + " 'Obrien') AND a.firstname = 'Mcgee'", - TEST_INDEX_PEOPLE, + TEST_INDEX_PEOPLE2, TEST_INDEX_ACCOUNT); JSONObject result = executeQuery(query); @@ -501,7 +503,7 @@ public void joinParseCheckSelectedFieldsSplitNLConditionOrderLT() throws IOExcep "SELECT /*! USE_NL*/ a.firstname, a.lastname, a.gender, d.firstname, d.age FROM %s a" + " JOIN %s d on a.age > d.age WHERE (d.firstname = 'Sandoval' OR d.firstname =" + " 'Hewitt') AND a.firstname = 'Fulton'", - TEST_INDEX_PEOPLE, + TEST_INDEX_PEOPLE2, TEST_INDEX_ACCOUNT); JSONObject result = executeQuery(query); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java index 62a6d63ef7..2b80575e1e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java @@ -5,13 +5,96 @@ package org.opensearch.sql.legacy.executor; +import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; +import static org.opensearch.search.sort.SortOrder.ASC; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.io.IOException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchRequestBuilder; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Client; +import org.opensearch.common.unit.TimeValue; import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.sql.legacy.domain.Select; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; + +/** Executor for search requests with pagination. */ +public abstract class ElasticHitsExecutor { + protected static final Logger LOG = LogManager.getLogger(); + protected PointInTimeHandler pit; + protected Client client; + + /** + * Executes search request + * + * @throws IOException If an input or output exception occurred + * @throws SqlParseException If parsing exception occurred + */ + protected abstract void run() throws IOException, SqlParseException; + + /** + * Get search hits after execution + * + * @return Search hits + */ + protected abstract SearchHits getHits(); + + /** + * Get response for search request with pit/scroll + * + * @param request search request + * @param select sql select + * @param size fetch size + * @param previousResponse response for previous request + * @param pit point in time + * @return search response for subsequent request + */ + public SearchResponse getResponseWithHits( + SearchRequestBuilder request, + Select select, + int size, + SearchResponse previousResponse, + PointInTimeHandler pit) { + // Set Size + request.setSize(size); + SearchResponse responseWithHits; -/** Created by Eliran on 21/8/2016. */ -public interface ElasticHitsExecutor { - void run() throws IOException, SqlParseException; + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + // Set sort field for search_after + boolean ordered = select.isOrderdSelect(); + if (!ordered) { + request.addSort(DOC_FIELD_NAME, ASC); + } + // Set PIT + request.setPointInTime(new PointInTimeBuilder(pit.getPitId())); + // from and size is alternate method to paginate result. + // If select has from clause, search after is not required. + if (previousResponse != null && select.getFrom().isEmpty()) { + request.searchAfter(previousResponse.getHits().getSortFields()); + } + responseWithHits = request.get(); + } else { + // Set scroll + TimeValue keepAlive = LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE); + if (previousResponse != null) { + responseWithHits = + client + .prepareSearchScroll(previousResponse.getScrollId()) + .setScroll(keepAlive) + .execute() + .actionGet(); + } else { + request.setScroll(keepAlive); + responseWithHits = request.get(); + } + } - SearchHits getHits(); + return responseWithHits; + } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java index f0ffafc470..061868c9b5 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -5,6 +5,8 @@ package org.opensearch.sql.legacy.executor.join; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.io.IOException; import java.util.Collection; import java.util.HashMap; @@ -12,15 +14,12 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import java.util.stream.Stream; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; -import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.unit.TimeValue; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperService; @@ -28,11 +27,11 @@ import org.opensearch.rest.RestChannel; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; -import org.opensearch.search.sort.FieldSortBuilder; -import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.legacy.domain.Field; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.ElasticHitsExecutor; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.SqlElasticRequestBuilder; import org.opensearch.sql.legacy.query.join.HashJoinElasticRequestBuilder; import org.opensearch.sql.legacy.query.join.JoinRequestBuilder; @@ -41,16 +40,16 @@ import org.opensearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; /** Created by Eliran on 15/9/2015. */ -public abstract class ElasticJoinExecutor implements ElasticHitsExecutor { - private static final Logger LOG = LogManager.getLogger(); +public abstract class ElasticJoinExecutor extends ElasticHitsExecutor { protected List results; // Keep list to avoid copy to new array in SearchHits protected MetaSearchResult metaResults; protected final int MAX_RESULTS_ON_ONE_FETCH = 10000; private Set aliasesOnReturn; private boolean allFieldsReturn; + protected String[] indices; - protected ElasticJoinExecutor(JoinRequestBuilder requestBuilder) { + protected ElasticJoinExecutor(Client client, JoinRequestBuilder requestBuilder) { metaResults = new MetaSearchResult(); aliasesOnReturn = new HashSet<>(); List firstTableReturnedField = requestBuilder.getFirstTable().getReturnedFields(); @@ -58,6 +57,8 @@ protected ElasticJoinExecutor(JoinRequestBuilder requestBuilder) { allFieldsReturn = (firstTableReturnedField == null || firstTableReturnedField.size() == 0) && (secondTableReturnedField == null || secondTableReturnedField.size() == 0); + indices = getIndices(requestBuilder); + this.client = client; } public void sendResponse(RestChannel channel) throws IOException { @@ -85,10 +86,22 @@ public void sendResponse(RestChannel channel) throws IOException { } public void run() throws IOException, SqlParseException { - long timeBefore = System.currentTimeMillis(); - results = innerRun(); - long joinTimeInMilli = System.currentTimeMillis() - timeBefore; - this.metaResults.setTookImMilli(joinTimeInMilli); + try { + long timeBefore = System.currentTimeMillis(); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit = new PointInTimeHandlerImpl(client, indices); + pit.create(); + } + results = innerRun(); + long joinTimeInMilli = System.currentTimeMillis() - timeBefore; + this.metaResults.setTookImMilli(joinTimeInMilli); + } catch (Exception e) { + LOG.error("Failed during join query run.", e); + } finally { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit.delete(); + } + } } protected abstract List innerRun() throws IOException, SqlParseException; @@ -103,7 +116,7 @@ public SearchHits getHits() { public static ElasticJoinExecutor createJoinExecutor( Client client, SqlElasticRequestBuilder requestBuilder) { if (requestBuilder instanceof HashJoinQueryPlanRequestBuilder) { - return new QueryPlanElasticExecutor((HashJoinQueryPlanRequestBuilder) requestBuilder); + return new QueryPlanElasticExecutor(client, (HashJoinQueryPlanRequestBuilder) requestBuilder); } else if (requestBuilder instanceof HashJoinElasticRequestBuilder) { HashJoinElasticRequestBuilder hashJoin = (HashJoinElasticRequestBuilder) requestBuilder; return new HashJoinElasticExecutor(client, hashJoin); @@ -256,23 +269,22 @@ protected void updateMetaSearchResults(SearchResponse searchResponse) { this.metaResults.updateTimeOut(searchResponse.isTimedOut()); } - protected SearchResponse scrollOneTimeWithMax( - Client client, TableInJoinRequestBuilder tableRequest) { - SearchRequestBuilder scrollRequest = - tableRequest - .getRequestBuilder() - .setScroll(new TimeValue(60000)) - .setSize(MAX_RESULTS_ON_ONE_FETCH); - boolean ordered = tableRequest.getOriginalSelect().isOrderdSelect(); - if (!ordered) { - scrollRequest.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); - } - SearchResponse responseWithHits = scrollRequest.get(); - // on ordered select - not using SCAN , elastic returns hits on first scroll - // es5.0 elastic always return docs on scan - // if(!ordered) - // responseWithHits = client.prepareSearchScroll(responseWithHits.getScrollId()) - // .setScroll(new TimeValue(600000)).get(); - return responseWithHits; + public SearchResponse getResponseWithHits( + TableInJoinRequestBuilder tableRequest, int size, SearchResponse previousResponse) { + + return getResponseWithHits( + tableRequest.getRequestBuilder(), + tableRequest.getOriginalSelect(), + size, + previousResponse, + pit); + } + + public String[] getIndices(JoinRequestBuilder joinRequestBuilder) { + return Stream.concat( + Stream.of(joinRequestBuilder.getFirstTable().getOriginalSelect().getIndexArr()), + Stream.of(joinRequestBuilder.getSecondTable().getOriginalSelect().getIndexArr())) + .distinct() + .toArray(String[]::new); } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java index 06a913205d..0e33ab9eef 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java @@ -20,7 +20,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.unit.TimeValue; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -36,16 +35,13 @@ /** Created by Eliran on 22/8/2015. */ public class HashJoinElasticExecutor extends ElasticJoinExecutor { private HashJoinElasticRequestBuilder requestBuilder; - - private Client client; private boolean useQueryTermsFilterOptimization = false; private final int MAX_RESULTS_FOR_FIRST_TABLE = 100000; HashJoinComparisonStructure hashJoinComparisonStructure; private Set alreadyMatched; public HashJoinElasticExecutor(Client client, HashJoinElasticRequestBuilder requestBuilder) { - super(requestBuilder); - this.client = client; + super(client, requestBuilder); this.requestBuilder = requestBuilder; this.useQueryTermsFilterOptimization = requestBuilder.isUseTermFiltersOptimization(); this.hashJoinComparisonStructure = @@ -54,7 +50,6 @@ public HashJoinElasticExecutor(Client client, HashJoinElasticRequestBuilder requ } public List innerRun() throws IOException, SqlParseException { - Map>> optimizationTermsFilterStructure = initOptimizationStructure(); @@ -124,16 +119,12 @@ private List createCombinedResults(TableInJoinRequestBuilder secondTa Integer hintLimit = secondTableRequest.getHintLimit(); SearchResponse searchResponse; boolean finishedScrolling; + if (hintLimit != null && hintLimit < MAX_RESULTS_ON_ONE_FETCH) { - searchResponse = secondTableRequest.getRequestBuilder().setSize(hintLimit).get(); + searchResponse = getResponseWithHits(secondTableRequest, hintLimit, null); finishedScrolling = true; } else { - searchResponse = - secondTableRequest - .getRequestBuilder() - .setScroll(new TimeValue(60000)) - .setSize(MAX_RESULTS_ON_ONE_FETCH) - .get(); + searchResponse = getResponseWithHits(secondTableRequest, MAX_RESULTS_ON_ONE_FETCH, null); // es5.0 no need to scroll again! // searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()) // .setScroll(new TimeValue(600000)).get(); @@ -214,11 +205,7 @@ private List createCombinedResults(TableInJoinRequestBuilder secondTa if (secondTableHits.length > 0 && (hintLimit == null || fetchedSoFarFromSecondTable >= hintLimit)) { searchResponse = - client - .prepareSearchScroll(searchResponse.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits(secondTableRequest, MAX_RESULTS_ON_ONE_FETCH, searchResponse); } else { break; } @@ -292,12 +279,13 @@ private List fetchAllHits(TableInJoinRequestBuilder tableInJoinReques private List scrollTillLimit( TableInJoinRequestBuilder tableInJoinRequest, Integer hintLimit) { - SearchResponse scrollResp = scrollOneTimeWithMax(client, tableInJoinRequest); + SearchResponse response = + getResponseWithHits(tableInJoinRequest, MAX_RESULTS_ON_ONE_FETCH, null); - updateMetaSearchResults(scrollResp); + updateMetaSearchResults(response); List hitsWithScan = new ArrayList<>(); int curentNumOfResults = 0; - SearchHit[] hits = scrollResp.getHits().getHits(); + SearchHit[] hits = response.getHits().getHits(); if (hintLimit == null) { hintLimit = MAX_RESULTS_FOR_FIRST_TABLE; @@ -311,13 +299,8 @@ private List scrollTillLimit( System.out.println("too many results for first table, stoping at:" + curentNumOfResults); break; } - scrollResp = - client - .prepareSearchScroll(scrollResp.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); - hits = scrollResp.getHits().getHits(); + response = getResponseWithHits(tableInJoinRequest, MAX_RESULTS_FOR_FIRST_TABLE, response); + hits = response.getHits().getHits(); } return hitsWithScan; } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java index 56c5f96af5..9356a0058e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java @@ -18,7 +18,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.unit.TimeValue; import org.opensearch.index.mapper.MapperService; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @@ -39,11 +38,9 @@ public class NestedLoopsElasticExecutor extends ElasticJoinExecutor { private static final Logger LOG = LogManager.getLogger(); private final NestedLoopsElasticRequestBuilder nestedLoopsRequest; - private final Client client; public NestedLoopsElasticExecutor(Client client, NestedLoopsElasticRequestBuilder nestedLoops) { - super(nestedLoops); - this.client = client; + super(client, nestedLoops); this.nestedLoopsRequest = nestedLoops; } @@ -111,11 +108,26 @@ protected List innerRun() throws SqlParseException { if (!BackOffRetryStrategy.isHealthy()) { throw new IllegalStateException("Memory circuit is broken"); } - firstTableResponse = - client - .prepareSearchScroll(firstTableResponse.getScrollId()) - .setScroll(new TimeValue(600000)) - .get(); + /* Fetching next result page. + Using scroll api - only scrollId from previous response is required for scroll request. + Using pit with search_after - we need to recreate search request along with pitId and + sort fields from previous response. + Here we are finding required size for recreating search request with pit and search after. + Conditions for size are similar as firstFetch(). + In case of scroll, this size will be ignored and size from first request will be used. + */ + Integer hintLimit = nestedLoopsRequest.getFirstTable().getHintLimit(); + if (hintLimit != null && hintLimit < MAX_RESULTS_ON_ONE_FETCH) { + firstTableResponse = + getResponseWithHits( + nestedLoopsRequest.getFirstTable(), hintLimit, firstTableResponse); + } else { + firstTableResponse = + getResponseWithHits( + nestedLoopsRequest.getFirstTable(), + MAX_RESULTS_ON_ONE_FETCH, + firstTableResponse); + } } else { finishedWithFirstTable = true; } @@ -287,12 +299,11 @@ private FetchWithScrollResponse firstFetch(TableInJoinRequestBuilder tableReques boolean needScrollForFirstTable = false; SearchResponse responseWithHits; if (hintLimit != null && hintLimit < MAX_RESULTS_ON_ONE_FETCH) { - responseWithHits = tableRequest.getRequestBuilder().setSize(hintLimit).get(); needScrollForFirstTable = false; } else { // scroll request with max. - responseWithHits = scrollOneTimeWithMax(client, tableRequest); + responseWithHits = getResponseWithHits(tableRequest, MAX_RESULTS_ON_ONE_FETCH, null); if (responseWithHits.getHits().getTotalHits() != null && responseWithHits.getHits().getTotalHits().value < MAX_RESULTS_ON_ONE_FETCH) { needScrollForFirstTable = true; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java index f4b2f5421d..d8e9d41376 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy.executor.join; import java.util.List; +import org.opensearch.client.Client; import org.opensearch.search.SearchHit; import org.opensearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; import org.opensearch.sql.legacy.query.planner.core.QueryPlanner; @@ -19,8 +20,8 @@ class QueryPlanElasticExecutor extends ElasticJoinExecutor { private final QueryPlanner queryPlanner; - QueryPlanElasticExecutor(HashJoinQueryPlanRequestBuilder request) { - super(request); + QueryPlanElasticExecutor(Client client, HashJoinQueryPlanRequestBuilder request) { + super(client, request); this.queryPlanner = request.plan(); } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java index 03e16424e7..f58b25e821 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java @@ -5,6 +5,8 @@ package org.opensearch.sql.legacy.executor.multi; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -18,7 +20,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.ArrayUtils; import org.opensearch.index.mapper.MapperService; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @@ -28,16 +30,16 @@ import org.opensearch.sql.legacy.domain.Where; import org.opensearch.sql.legacy.domain.hints.Hint; import org.opensearch.sql.legacy.domain.hints.HintType; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.ElasticHitsExecutor; -import org.opensearch.sql.legacy.executor.join.ElasticUtils; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.DefaultQueryAction; import org.opensearch.sql.legacy.query.multi.MultiQueryRequestBuilder; import org.opensearch.sql.legacy.utils.Util; /** Created by Eliran on 26/8/2016. */ -public class MinusExecutor implements ElasticHitsExecutor { - private Client client; +public class MinusExecutor extends ElasticHitsExecutor { private MultiQueryRequestBuilder builder; private SearchHits minusHits; private boolean useTermsOptimization; @@ -63,45 +65,63 @@ public MinusExecutor(Client client, MultiQueryRequestBuilder builder) { @Override public void run() throws SqlParseException { - if (this.useTermsOptimization && this.fieldsOrderFirstTable.length != 1) { - throw new SqlParseException( - "Terms optimization failed: terms optimization for minus execution is supported with one" - + " field"); - } - if (this.useTermsOptimization && !this.useScrolling) { - throw new SqlParseException( - "Terms optimization failed: using scrolling is required for terms optimization"); - } - if (!this.useScrolling || !this.useTermsOptimization) { - Set comperableHitResults; - if (!this.useScrolling) { - // 1. get results from first search , put in set - // 2. get reults from second search - // 2.1 for each result remove from set - comperableHitResults = simpleOneTimeQueryEach(); + try { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit = + new PointInTimeHandlerImpl( + client, + ArrayUtils.concat( + builder.getOriginalSelect(true).getIndexArr(), + builder.getOriginalSelect(false).getIndexArr())); + pit.create(); + } + + if (this.useTermsOptimization && this.fieldsOrderFirstTable.length != 1) { + throw new SqlParseException( + "Terms optimization failed: terms optimization for minus execution is supported with" + + " one field"); + } + if (this.useTermsOptimization && !this.useScrolling) { + throw new SqlParseException( + "Terms optimization failed: using scrolling is required for terms optimization"); + } + if (!this.useScrolling || !this.useTermsOptimization) { + Set comperableHitResults; + if (!this.useScrolling) { + // 1. get results from first search , put in set + // 2. get reults from second search + // 2.1 for each result remove from set + comperableHitResults = simpleOneTimeQueryEach(); + } else { + // if scrolling + // 1. get all results in scrolls (till some limit) . put on set + // 2. scroll on second table + // 3. on each scroll result remove items from set + comperableHitResults = runWithScrollings(); + } + fillMinusHitsFromResults(comperableHitResults); + return; } else { - // if scrolling - // 1. get all results in scrolls (till some limit) . put on set - // 2. scroll on second table - // 3. on each scroll result remove items from set - comperableHitResults = runWithScrollings(); + // if scrolling and optimization + // 0. save the original second table where , init set + // 1. on each scroll on first table , create miniSet + // 1.1 build where from all results (terms filter) , and run query + // 1.1.1 on each result remove from miniSet + // 1.1.2 add all results left from miniset to bigset + Select firstSelect = this.builder.getOriginalSelect(true); + MinusOneFieldAndOptimizationResult optimizationResult = + runWithScrollingAndAddFilter(fieldsOrderFirstTable[0], fieldsOrderSecondTable[0]); + String fieldName = getFieldName(firstSelect.getFields().get(0)); + Set results = optimizationResult.getFieldValues(); + SearchHit someHit = optimizationResult.getSomeHit(); + fillMinusHitsFromOneField(fieldName, results, someHit); + } + } catch (Exception e) { + LOG.error("Failed during multi query run.", e); + } finally { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit.delete(); } - fillMinusHitsFromResults(comperableHitResults); - return; - } else { - // if scrolling and optimization - // 0. save the original second table where , init set - // 1. on each scroll on first table , create miniSet - // 1.1 build where from all results (terms filter) , and run query - // 1.1.1 on each result remove from miniSet - // 1.1.2 add all results left from miniset to bigset - Select firstSelect = this.builder.getOriginalSelect(true); - MinusOneFieldAndOptimizationResult optimizationResult = - runWithScrollingAndAddFilter(fieldsOrderFirstTable[0], fieldsOrderSecondTable[0]); - String fieldName = getFieldName(firstSelect.getFields().get(0)); - Set results = optimizationResult.getFieldValues(); - SearchHit someHit = optimizationResult.getSomeHit(); - fillMinusHitsFromOneField(fieldName, results, someHit); } } @@ -187,11 +207,12 @@ private void fillMinusHitsFromResults(Set comperableHitResu private Set runWithScrollings() { SearchResponse scrollResp = - ElasticUtils.scrollOneTimeWithHits( - this.client, - this.builder.getFirstSearchRequest(), + getResponseWithHits( + builder.getFirstSearchRequest(), builder.getOriginalSelect(true), - this.maxDocsToFetchOnEachScrollShard); + maxDocsToFetchOnEachScrollShard, + null, + pit); Set results = new HashSet<>(); SearchHit[] hits = scrollResp.getHits().getHits(); @@ -199,7 +220,6 @@ private Set runWithScrollings() { return new HashSet<>(); } int totalDocsFetchedFromFirstTable = 0; - // fetch from first table . fill set. while (hits != null && hits.length != 0) { totalDocsFetchedFromFirstTable += hits.length; @@ -208,19 +228,21 @@ private Set runWithScrollings() { break; } scrollResp = - client - .prepareSearchScroll(scrollResp.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits( + builder.getFirstSearchRequest(), + builder.getOriginalSelect(true), + maxDocsToFetchOnEachScrollShard, + scrollResp, + pit); hits = scrollResp.getHits().getHits(); } scrollResp = - ElasticUtils.scrollOneTimeWithHits( - this.client, + getResponseWithHits( this.builder.getSecondSearchRequest(), builder.getOriginalSelect(false), - this.maxDocsToFetchOnEachScrollShard); + this.maxDocsToFetchOnEachScrollShard, + null, + pit); hits = scrollResp.getHits().getHits(); if (hits == null || hits.length == 0) { @@ -234,11 +256,12 @@ private Set runWithScrollings() { break; } scrollResp = - client - .prepareSearchScroll(scrollResp.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits( + builder.getSecondSearchRequest(), + builder.getOriginalSelect(false), + maxDocsToFetchOnEachScrollShard, + scrollResp, + pit); hits = scrollResp.getHits().getHits(); } @@ -303,11 +326,12 @@ private boolean checkIfOnlyOneField(Select firstSelect, Select secondSelect) { private MinusOneFieldAndOptimizationResult runWithScrollingAndAddFilter( String firstFieldName, String secondFieldName) throws SqlParseException { SearchResponse scrollResp = - ElasticUtils.scrollOneTimeWithHits( - this.client, - this.builder.getFirstSearchRequest(), + getResponseWithHits( + builder.getFirstSearchRequest(), builder.getOriginalSelect(true), - this.maxDocsToFetchOnEachScrollShard); + maxDocsToFetchOnEachScrollShard, + null, + pit); Set results = new HashSet<>(); int currentNumOfResults = 0; SearchHit[] hits = scrollResp.getHits().getHits(); @@ -335,14 +359,16 @@ private MinusOneFieldAndOptimizationResult runWithScrollingAndAddFilter( break; } SearchResponse responseForSecondTable = - ElasticUtils.scrollOneTimeWithHits( - this.client, + getResponseWithHits( queryAction.getRequestBuilder(), secondQuerySelect, - this.maxDocsToFetchOnEachScrollShard); + this.maxDocsToFetchOnEachScrollShard, + null, + pit); SearchHits secondQuerySearchHits = responseForSecondTable.getHits(); SearchHit[] secondQueryHits = secondQuerySearchHits.getHits(); + while (secondQueryHits.length > 0) { totalDocsFetchedFromSecondTable += secondQueryHits.length; removeValuesFromSetAccordingToHits(secondFieldName, currentSetFromResults, secondQueryHits); @@ -350,11 +376,12 @@ private MinusOneFieldAndOptimizationResult runWithScrollingAndAddFilter( break; } responseForSecondTable = - client - .prepareSearchScroll(responseForSecondTable.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits( + queryAction.getRequestBuilder(), + secondQuerySelect, + maxDocsToFetchOnEachScrollShard, + responseForSecondTable, + pit); secondQueryHits = responseForSecondTable.getHits().getHits(); } results.addAll(currentSetFromResults); @@ -363,13 +390,13 @@ private MinusOneFieldAndOptimizationResult runWithScrollingAndAddFilter( "too many results for first table, stoping at:" + totalDocsFetchedFromFirstTable); break; } - scrollResp = - client - .prepareSearchScroll(scrollResp.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits( + builder.getFirstSearchRequest(), + builder.getOriginalSelect(true), + maxDocsToFetchOnEachScrollShard, + scrollResp, + pit); hits = scrollResp.getHits().getHits(); } return new MinusOneFieldAndOptimizationResult(results, someHit); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java index 6b8b64c4e8..375c40a5c1 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java @@ -23,11 +23,10 @@ import org.opensearch.sql.legacy.utils.Util; /** Created by Eliran on 21/8/2016. */ -public class UnionExecutor implements ElasticHitsExecutor { +public class UnionExecutor extends ElasticHitsExecutor { private MultiQueryRequestBuilder multiQueryBuilder; private SearchHits results; - private Client client; private int currentId; public UnionExecutor(Client client, MultiQueryRequestBuilder builder) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandler.java b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandler.java new file mode 100644 index 0000000000..66339cc70a --- /dev/null +++ b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandler.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy.pit; + +/** Point In Time */ +public interface PointInTimeHandler { + /** Create Point In Time */ + void create(); + + /** Delete Point In Time */ + void delete(); + + /** Get Point In Time Identifier */ + String getPitId(); +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java new file mode 100644 index 0000000000..64535749e8 --- /dev/null +++ b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy.pit; + +import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; + +import lombok.Getter; +import lombok.Setter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.CreatePitRequest; +import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.DeletePitRequest; +import org.opensearch.action.search.DeletePitResponse; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; + +/** Handler for Point In Time */ +public class PointInTimeHandlerImpl implements PointInTimeHandler { + private Client client; + private String[] indices; + @Getter @Setter private String pitId; + private static final Logger LOG = LogManager.getLogger(); + + /** + * Constructor for class + * + * @param client OpenSearch client + * @param indices list of indices + */ + public PointInTimeHandlerImpl(Client client, String[] indices) { + this.client = client; + this.indices = indices; + } + + /** Create PIT for given indices */ + @Override + public void create() { + CreatePitRequest createPitRequest = + new CreatePitRequest( + LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE), false, indices); + client.createPit( + createPitRequest, + new ActionListener<>() { + @Override + public void onResponse(CreatePitResponse createPitResponse) { + pitId = createPitResponse.getId(); + LOG.info("Created Point In Time {} successfully.", pitId); + } + + @Override + public void onFailure(Exception e) { + LOG.error("Error occurred while creating PIT", e); + } + }); + } + + /** Delete PIT */ + @Override + public void delete() { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + client.deletePits( + deletePitRequest, + new ActionListener<>() { + @Override + public void onResponse(DeletePitResponse deletePitResponse) { + LOG.info( + "Delete Point In Time {} status: {}", + pitId, + deletePitResponse.status().getStatus()); + } + + @Override + public void onFailure(Exception e) { + LOG.error("Error occurred while deleting PIT", e); + } + }); + } +} diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java new file mode 100644 index 0000000000..42f1af4563 --- /dev/null +++ b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy.pit; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; + +import java.util.concurrent.CompletableFuture; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.DeletePitResponse; +import org.opensearch.client.Client; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; + +public class PointInTimeHandlerImplTest { + + @Mock private Client mockClient; + private String[] indices = {"index1", "index2"}; + private PointInTimeHandlerImpl pointInTimeHandlerImpl; + @Captor private ArgumentCaptor> listenerCaptor; + private final String PIT_ID = "testId"; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + pointInTimeHandlerImpl = new PointInTimeHandlerImpl(mockClient, indices); + } + + @Test + public void testCreate() { + when(LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE)) + .thenReturn(new TimeValue(10000)); + + CreatePitResponse mockCreatePitResponse = mock(CreatePitResponse.class); + when(mockCreatePitResponse.getId()).thenReturn(PIT_ID); + + CompletableFuture completableFuture = + CompletableFuture.completedFuture(mockCreatePitResponse); + + doAnswer( + invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(mockCreatePitResponse); + return completableFuture; + }) + .when(mockClient) + .createPit(any(), any()); + + pointInTimeHandlerImpl.create(); + + assertEquals(PIT_ID, pointInTimeHandlerImpl.getPitId()); + } + + @Test + public void testDelete() { + DeletePitResponse mockedResponse = mock(DeletePitResponse.class); + RestStatus mockRestStatus = mock(RestStatus.class); + when(mockedResponse.status()).thenReturn(mockRestStatus); + when(mockedResponse.status().getStatus()).thenReturn(200); + pointInTimeHandlerImpl.setPitId(PIT_ID); + pointInTimeHandlerImpl.delete(); + verify(mockClient).deletePits(any(), listenerCaptor.capture()); + listenerCaptor.getValue().onResponse(mockedResponse); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 3cb245b487..1083dbd836 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -71,6 +71,13 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting SQL_PAGINATION_API_SEARCH_AFTER_SETTING = + Setting.boolSetting( + Key.SQL_PAGINATION_API_SEARCH_AFTER.getKeyValue(), + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + public static final Setting PPL_ENABLED_SETTING = Setting.boolSetting( Key.PPL_ENABLED.getKeyValue(), @@ -250,6 +257,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.SQL_DELETE_ENABLED, SQL_DELETE_ENABLED_SETTING, new Updater(Key.SQL_DELETE_ENABLED)); + register( + settingBuilder, + clusterSettings, + Key.SQL_PAGINATION_API_SEARCH_AFTER, + SQL_PAGINATION_API_SEARCH_AFTER_SETTING, + new Updater(Key.SQL_PAGINATION_API_SEARCH_AFTER)); register( settingBuilder, clusterSettings, @@ -422,6 +435,7 @@ public static List> pluginSettings() { .add(SQL_SLOWLOG_SETTING) .add(SQL_CURSOR_KEEP_ALIVE_SETTING) .add(SQL_DELETE_ENABLED_SETTING) + .add(SQL_PAGINATION_API_SEARCH_AFTER_SETTING) .add(PPL_ENABLED_SETTING) .add(QUERY_MEMORY_LIMIT_SETTING) .add(QUERY_SIZE_LIMIT_SETTING) From e679df5a6788b27d232273230b845c8948a256d5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:08:54 -0700 Subject: [PATCH 09/32] [Backport 2.x] Refer Antlr file using specific branch (#3054) * Refer Antlr file using specific branch * Fix version to current one * Disable auto download * Disable auto download --------- (cherry picked from commit d7710d09593ee66f5efddb0c546f539fdcaa598a) Signed-off-by: Tomoyuki Morita Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- async-query-core/README.md | 8 + async-query-core/build.gradle | 16 +- .../src/main/antlr/SqlBaseLexer.g4 | 89 +---- .../src/main/antlr/SqlBaseParser.g4 | 373 +++--------------- 4 files changed, 87 insertions(+), 399 deletions(-) diff --git a/async-query-core/README.md b/async-query-core/README.md index 815088bce6..08301c024d 100644 --- a/async-query-core/README.md +++ b/async-query-core/README.md @@ -32,3 +32,11 @@ Following is the list of extension points where the consumer of the library need - [DataSourceSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/DataSourceSparkParameterComposer.java) - [GeneralSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/GeneralSparkParameterComposer.java) - [SparkSubmitParameterModifier](src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java) To be deprecated in favor of GeneralSparkParameterComposer + +## Update Grammar files +This package uses ANTLR grammar files from `opensearch-spark` and `Spark` repositories. +To update the grammar files, update `build.gradle` file (in `downloadG4Files` task) as needed and run: +``` +./gradlew async-query-core:downloadG4Files +``` +This will overwrite the files under `src/main/antlr`. \ No newline at end of file diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index deba81735d..330b418681 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -21,10 +21,12 @@ tasks.register('downloadG4Files', Exec) { executable 'curl' - args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4' - args '-o', 'src/main/antlr/SparkSqlBase.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4' - args '-o', 'src/main/antlr/SqlBaseParser.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4' - args '-o', 'src/main/antlr/SqlBaseLexer.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4' + def opensearchSparkBranch = "0.5" + def apacheSparkVersionTag = "v3.5.1" + args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4" + args '-o', 'src/main/antlr/SparkSqlBase.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4" + args '-o', 'src/main/antlr/SqlBaseParser.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4" + args '-o', 'src/main/antlr/SqlBaseLexer.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4" } generateGrammarSource { @@ -38,12 +40,6 @@ configurations { } } -// skip download in case of offline build -if (!gradle.startParameter.offline) { - // Make sure the downloadG4File task runs before the generateGrammarSource task - generateGrammarSource.dependsOn downloadG4Files -} - dependencies { antlr "org.antlr:antlr4:4.7.1" diff --git a/async-query-core/src/main/antlr/SqlBaseLexer.g4 b/async-query-core/src/main/antlr/SqlBaseLexer.g4 index acfc0011f5..fb440ef8d3 100644 --- a/async-query-core/src/main/antlr/SqlBaseLexer.g4 +++ b/async-query-core/src/main/antlr/SqlBaseLexer.g4 @@ -69,35 +69,6 @@ lexer grammar SqlBaseLexer; public void markUnclosedComment() { has_unclosed_bracketed_comment = true; } - - /** - * When greater than zero, it's in the middle of parsing ARRAY/MAP/STRUCT type. - */ - public int complex_type_level_counter = 0; - - /** - * Increase the counter by one when hits KEYWORD 'ARRAY', 'MAP', 'STRUCT'. - */ - public void incComplexTypeLevelCounter() { - complex_type_level_counter++; - } - - /** - * Decrease the counter by one when hits close tag '>' && the counter greater than zero - * which means we are in the middle of complex type parsing. Otherwise, it's a dangling - * GT token and we do nothing. - */ - public void decComplexTypeLevelCounter() { - if (complex_type_level_counter > 0) complex_type_level_counter--; - } - - /** - * If the counter is zero, it's a shift right operator. It can be closing tags of an complex - * type definition, such as MAP>. - */ - public boolean isShiftRightOperator() { - return complex_type_level_counter == 0 ? true : false; - } } SEMICOLON: ';'; @@ -108,7 +79,6 @@ COMMA: ','; DOT: '.'; LEFT_BRACKET: '['; RIGHT_BRACKET: ']'; -BANG: '!'; // NOTE: If you add a new token in the list below, you should update the list of keywords // and reserved tag in `docs/sql-ref-ansi-compliance.md#sql-keywords`, and @@ -129,16 +99,14 @@ ANTI: 'ANTI'; ANY: 'ANY'; ANY_VALUE: 'ANY_VALUE'; ARCHIVE: 'ARCHIVE'; -ARRAY: 'ARRAY' {incComplexTypeLevelCounter();}; +ARRAY: 'ARRAY'; AS: 'AS'; ASC: 'ASC'; AT: 'AT'; AUTHORIZATION: 'AUTHORIZATION'; -BEGIN: 'BEGIN'; BETWEEN: 'BETWEEN'; BIGINT: 'BIGINT'; BINARY: 'BINARY'; -BINDING: 'BINDING'; BOOLEAN: 'BOOLEAN'; BOTH: 'BOTH'; BUCKET: 'BUCKET'; @@ -146,7 +114,6 @@ BUCKETS: 'BUCKETS'; BY: 'BY'; BYTE: 'BYTE'; CACHE: 'CACHE'; -CALLED: 'CALLED'; CASCADE: 'CASCADE'; CASE: 'CASE'; CAST: 'CAST'; @@ -161,7 +128,6 @@ CLUSTER: 'CLUSTER'; CLUSTERED: 'CLUSTERED'; CODEGEN: 'CODEGEN'; COLLATE: 'COLLATE'; -COLLATION: 'COLLATION'; COLLECTION: 'COLLECTION'; COLUMN: 'COLUMN'; COLUMNS: 'COLUMNS'; @@ -169,11 +135,9 @@ COMMENT: 'COMMENT'; COMMIT: 'COMMIT'; COMPACT: 'COMPACT'; COMPACTIONS: 'COMPACTIONS'; -COMPENSATION: 'COMPENSATION'; COMPUTE: 'COMPUTE'; CONCATENATE: 'CONCATENATE'; CONSTRAINT: 'CONSTRAINT'; -CONTAINS: 'CONTAINS'; COST: 'COST'; CREATE: 'CREATE'; CROSS: 'CROSS'; @@ -197,29 +161,24 @@ DATE_DIFF: 'DATE_DIFF'; DBPROPERTIES: 'DBPROPERTIES'; DEC: 'DEC'; DECIMAL: 'DECIMAL'; -DECLARE: 'DECLARE'; DEFAULT: 'DEFAULT'; DEFINED: 'DEFINED'; -DEFINER: 'DEFINER'; DELETE: 'DELETE'; DELIMITED: 'DELIMITED'; DESC: 'DESC'; DESCRIBE: 'DESCRIBE'; -DETERMINISTIC: 'DETERMINISTIC'; DFS: 'DFS'; DIRECTORIES: 'DIRECTORIES'; DIRECTORY: 'DIRECTORY'; DISTINCT: 'DISTINCT'; DISTRIBUTE: 'DISTRIBUTE'; DIV: 'DIV'; -DO: 'DO'; DOUBLE: 'DOUBLE'; DROP: 'DROP'; ELSE: 'ELSE'; END: 'END'; ESCAPE: 'ESCAPE'; ESCAPED: 'ESCAPED'; -EVOLUTION: 'EVOLUTION'; EXCEPT: 'EXCEPT'; EXCHANGE: 'EXCHANGE'; EXCLUDE: 'EXCLUDE'; @@ -257,7 +216,6 @@ HOURS: 'HOURS'; IDENTIFIER_KW: 'IDENTIFIER'; IF: 'IF'; IGNORE: 'IGNORE'; -IMMEDIATE: 'IMMEDIATE'; IMPORT: 'IMPORT'; IN: 'IN'; INCLUDE: 'INCLUDE'; @@ -265,7 +223,6 @@ INDEX: 'INDEX'; INDEXES: 'INDEXES'; INNER: 'INNER'; INPATH: 'INPATH'; -INPUT: 'INPUT'; INPUTFORMAT: 'INPUTFORMAT'; INSERT: 'INSERT'; INTERSECT: 'INTERSECT'; @@ -273,12 +230,10 @@ INTERVAL: 'INTERVAL'; INT: 'INT'; INTEGER: 'INTEGER'; INTO: 'INTO'; -INVOKER: 'INVOKER'; IS: 'IS'; ITEMS: 'ITEMS'; JOIN: 'JOIN'; KEYS: 'KEYS'; -LANGUAGE: 'LANGUAGE'; LAST: 'LAST'; LATERAL: 'LATERAL'; LAZY: 'LAZY'; @@ -297,7 +252,7 @@ LOCKS: 'LOCKS'; LOGICAL: 'LOGICAL'; LONG: 'LONG'; MACRO: 'MACRO'; -MAP: 'MAP' {incComplexTypeLevelCounter();}; +MAP: 'MAP'; MATCHED: 'MATCHED'; MERGE: 'MERGE'; MICROSECOND: 'MICROSECOND'; @@ -306,7 +261,6 @@ MILLISECOND: 'MILLISECOND'; MILLISECONDS: 'MILLISECONDS'; MINUTE: 'MINUTE'; MINUTES: 'MINUTES'; -MODIFIES: 'MODIFIES'; MONTH: 'MONTH'; MONTHS: 'MONTHS'; MSCK: 'MSCK'; @@ -317,8 +271,7 @@ NANOSECOND: 'NANOSECOND'; NANOSECONDS: 'NANOSECONDS'; NATURAL: 'NATURAL'; NO: 'NO'; -NONE: 'NONE'; -NOT: 'NOT'; +NOT: 'NOT' | '!'; NULL: 'NULL'; NULLS: 'NULLS'; NUMERIC: 'NUMERIC'; @@ -340,6 +293,8 @@ OVERWRITE: 'OVERWRITE'; PARTITION: 'PARTITION'; PARTITIONED: 'PARTITIONED'; PARTITIONS: 'PARTITIONS'; +PERCENTILE_CONT: 'PERCENTILE_CONT'; +PERCENTILE_DISC: 'PERCENTILE_DISC'; PERCENTLIT: 'PERCENT'; PIVOT: 'PIVOT'; PLACING: 'PLACING'; @@ -352,7 +307,6 @@ PURGE: 'PURGE'; QUARTER: 'QUARTER'; QUERY: 'QUERY'; RANGE: 'RANGE'; -READS: 'READS'; REAL: 'REAL'; RECORDREADER: 'RECORDREADER'; RECORDWRITER: 'RECORDWRITER'; @@ -367,8 +321,6 @@ REPLACE: 'REPLACE'; RESET: 'RESET'; RESPECT: 'RESPECT'; RESTRICT: 'RESTRICT'; -RETURN: 'RETURN'; -RETURNS: 'RETURNS'; REVOKE: 'REVOKE'; RIGHT: 'RIGHT'; RLIKE: 'RLIKE' | 'REGEXP'; @@ -382,7 +334,6 @@ SECOND: 'SECOND'; SECONDS: 'SECONDS'; SCHEMA: 'SCHEMA'; SCHEMAS: 'SCHEMAS'; -SECURITY: 'SECURITY'; SELECT: 'SELECT'; SEMI: 'SEMI'; SEPARATED: 'SEPARATED'; @@ -394,21 +345,18 @@ SETMINUS: 'MINUS'; SETS: 'SETS'; SHORT: 'SHORT'; SHOW: 'SHOW'; -SINGLE: 'SINGLE'; SKEWED: 'SKEWED'; SMALLINT: 'SMALLINT'; SOME: 'SOME'; SORT: 'SORT'; SORTED: 'SORTED'; SOURCE: 'SOURCE'; -SPECIFIC: 'SPECIFIC'; -SQL: 'SQL'; START: 'START'; STATISTICS: 'STATISTICS'; STORED: 'STORED'; STRATIFY: 'STRATIFY'; STRING: 'STRING'; -STRUCT: 'STRUCT' {incComplexTypeLevelCounter();}; +STRUCT: 'STRUCT'; SUBSTR: 'SUBSTR'; SUBSTRING: 'SUBSTRING'; SYNC: 'SYNC'; @@ -423,7 +371,6 @@ TEMPORARY: 'TEMPORARY' | 'TEMP'; TERMINATED: 'TERMINATED'; THEN: 'THEN'; TIME: 'TIME'; -TIMEDIFF: 'TIMEDIFF'; TIMESTAMP: 'TIMESTAMP'; TIMESTAMP_LTZ: 'TIMESTAMP_LTZ'; TIMESTAMP_NTZ: 'TIMESTAMP_NTZ'; @@ -431,7 +378,6 @@ TIMESTAMPADD: 'TIMESTAMPADD'; TIMESTAMPDIFF: 'TIMESTAMPDIFF'; TINYINT: 'TINYINT'; TO: 'TO'; -EXECUTE: 'EXECUTE'; TOUCH: 'TOUCH'; TRAILING: 'TRAILING'; TRANSACTION: 'TRANSACTION'; @@ -457,9 +403,6 @@ USER: 'USER'; USING: 'USING'; VALUES: 'VALUES'; VARCHAR: 'VARCHAR'; -VAR: 'VAR'; -VARIABLE: 'VARIABLE'; -VARIANT: 'VARIANT'; VERSION: 'VERSION'; VIEW: 'VIEW'; VIEWS: 'VIEWS'; @@ -468,7 +411,6 @@ WEEK: 'WEEK'; WEEKS: 'WEEKS'; WHEN: 'WHEN'; WHERE: 'WHERE'; -WHILE: 'WHILE'; WINDOW: 'WINDOW'; WITH: 'WITH'; WITHIN: 'WITHIN'; @@ -486,11 +428,8 @@ NEQ : '<>'; NEQJ: '!='; LT : '<'; LTE : '<=' | '!>'; -GT : '>' {decComplexTypeLevelCounter();}; +GT : '>'; GTE : '>=' | '!<'; -SHIFT_LEFT: '<<'; -SHIFT_RIGHT: '>>' {isShiftRightOperator()}?; -SHIFT_RIGHT_UNSIGNED: '>>>' {isShiftRightOperator()}?; PLUS: '+'; MINUS: '-'; @@ -503,7 +442,6 @@ PIPE: '|'; CONCAT_PIPE: '||'; HAT: '^'; COLON: ':'; -DOUBLE_COLON: '::'; ARROW: '->'; FAT_ARROW : '=>'; HENT_START: '/*+'; @@ -563,13 +501,8 @@ BIGDECIMAL_LITERAL | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}? ; -// Generalize the identifier to give a sensible INVALID_IDENTIFIER error message: -// * Unicode letters rather than a-z and A-Z only -// * URI paths for table references using paths -// We then narrow down to ANSI rules in exitUnquotedIdentifier() in the parser. IDENTIFIER - : (UNICODE_LETTER | DIGIT | '_')+ - | UNICODE_LETTER+ '://' (UNICODE_LETTER | DIGIT | '_' | '/' | '-' | '.' | '?' | '=' | '&' | '#' | '%')+ + : (LETTER | DIGIT | '_')+ ; BACKQUOTED_IDENTIFIER @@ -593,10 +526,6 @@ fragment LETTER : [A-Z] ; -fragment UNICODE_LETTER - : [\p{L}] - ; - SIMPLE_COMMENT : '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN) ; @@ -606,7 +535,7 @@ BRACKETED_COMMENT ; WS - : [ \t\n\f\r\u000B\u00A0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u202F\u205F\u3000]+ -> channel(HIDDEN) + : [ \r\n\t]+ -> channel(HIDDEN) ; // Catch-all for anything we can't recognize. diff --git a/async-query-core/src/main/antlr/SqlBaseParser.g4 b/async-query-core/src/main/antlr/SqlBaseParser.g4 index 5b8805821b..04128216be 100644 --- a/async-query-core/src/main/antlr/SqlBaseParser.g4 +++ b/async-query-core/src/main/antlr/SqlBaseParser.g4 @@ -42,57 +42,8 @@ options { tokenVocab = SqlBaseLexer; } public boolean double_quoted_identifiers = false; } -compoundOrSingleStatement - : singleStatement - | singleCompoundStatement - ; - -singleCompoundStatement - : beginEndCompoundBlock SEMICOLON? EOF - ; - -beginEndCompoundBlock - : beginLabel? BEGIN compoundBody END endLabel? - ; - -compoundBody - : (compoundStatements+=compoundStatement SEMICOLON)* - ; - -compoundStatement - : statement - | setStatementWithOptionalVarKeyword - | beginEndCompoundBlock - | ifElseStatement - | whileStatement - ; - -setStatementWithOptionalVarKeyword - : SET variable? assignmentList #setVariableWithOptionalKeyword - | SET variable? LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ - LEFT_PAREN query RIGHT_PAREN #setVariableWithOptionalKeyword - ; - -whileStatement - : beginLabel? WHILE booleanExpression DO compoundBody END WHILE endLabel? - ; - -ifElseStatement - : IF booleanExpression THEN conditionalBodies+=compoundBody - (ELSE IF booleanExpression THEN conditionalBodies+=compoundBody)* - (ELSE elseBody=compoundBody)? END IF - ; - singleStatement - : (statement|setResetStatement) SEMICOLON* EOF - ; - -beginLabel - : multipartIdentifier COLON - ; - -endLabel - : multipartIdentifier + : statement SEMICOLON* EOF ; singleExpression @@ -121,36 +72,33 @@ singleTableSchema statement : query #statementDefault - | executeImmediate #visitExecuteImmediate | ctes? dmlStatementNoWith #dmlStatement | USE identifierReference #use | USE namespace identifierReference #useNamespace - | SET CATALOG (errorCapturingIdentifier | stringLit) #setCatalog - | CREATE namespace (IF errorCapturingNot EXISTS)? identifierReference + | SET CATALOG (identifier | stringLit) #setCatalog + | CREATE namespace (IF NOT EXISTS)? identifierReference (commentSpec | locationSpec | (WITH (DBPROPERTIES | PROPERTIES) propertyList))* #createNamespace | ALTER namespace identifierReference SET (DBPROPERTIES | PROPERTIES) propertyList #setNamespaceProperties - | ALTER namespace identifierReference - UNSET (DBPROPERTIES | PROPERTIES) propertyList #unsetNamespaceProperties | ALTER namespace identifierReference SET locationSpec #setNamespaceLocation | DROP namespace (IF EXISTS)? identifierReference (RESTRICT | CASCADE)? #dropNamespace | SHOW namespaces ((FROM | IN) multipartIdentifier)? (LIKE? pattern=stringLit)? #showNamespaces - | createTableHeader (LEFT_PAREN colDefinitionList RIGHT_PAREN)? tableProvider? + | createTableHeader (LEFT_PAREN createOrReplaceTableColTypeList RIGHT_PAREN)? tableProvider? createTableClauses (AS? query)? #createTable - | CREATE TABLE (IF errorCapturingNot EXISTS)? target=tableIdentifier + | CREATE TABLE (IF NOT EXISTS)? target=tableIdentifier LIKE source=tableIdentifier (tableProvider | rowFormat | createFileFormat | locationSpec | (TBLPROPERTIES tableProps=propertyList))* #createTableLike - | replaceTableHeader (LEFT_PAREN colDefinitionList RIGHT_PAREN)? tableProvider? + | replaceTableHeader (LEFT_PAREN createOrReplaceTableColTypeList RIGHT_PAREN)? tableProvider? createTableClauses (AS? query)? #replaceTable | ANALYZE TABLE identifierReference partitionSpec? COMPUTE STATISTICS @@ -192,7 +140,7 @@ statement SET SERDE stringLit (WITH SERDEPROPERTIES propertyList)? #setTableSerDe | ALTER TABLE identifierReference (partitionSpec)? SET SERDEPROPERTIES propertyList #setTableSerDe - | ALTER (TABLE | VIEW) identifierReference ADD (IF errorCapturingNot EXISTS)? + | ALTER (TABLE | VIEW) identifierReference ADD (IF NOT EXISTS)? partitionSpecLocation+ #addTablePartition | ALTER TABLE identifierReference from=partitionSpec RENAME TO to=partitionSpec #renameTablePartition @@ -201,15 +149,12 @@ statement | ALTER TABLE identifierReference (partitionSpec)? SET locationSpec #setTableLocation | ALTER TABLE identifierReference RECOVER PARTITIONS #recoverPartitions - | ALTER TABLE identifierReference - (clusterBySpec | CLUSTER BY NONE) #alterClusterBy | DROP TABLE (IF EXISTS)? identifierReference PURGE? #dropTable | DROP VIEW (IF EXISTS)? identifierReference #dropView | CREATE (OR REPLACE)? (GLOBAL? TEMPORARY)? - VIEW (IF errorCapturingNot EXISTS)? identifierReference + VIEW (IF NOT EXISTS)? identifierReference identifierCommentList? (commentSpec | - schemaBinding | (PARTITIONED ON identifierList) | (TBLPROPERTIES propertyList))* AS query #createView @@ -217,21 +162,12 @@ statement tableIdentifier (LEFT_PAREN colTypeList RIGHT_PAREN)? tableProvider (OPTIONS propertyList)? #createTempViewUsing | ALTER VIEW identifierReference AS? query #alterViewQuery - | ALTER VIEW identifierReference schemaBinding #alterViewSchemaBinding - | CREATE (OR REPLACE)? TEMPORARY? FUNCTION (IF errorCapturingNot EXISTS)? + | CREATE (OR REPLACE)? TEMPORARY? FUNCTION (IF NOT EXISTS)? identifierReference AS className=stringLit (USING resource (COMMA resource)*)? #createFunction - | CREATE (OR REPLACE)? TEMPORARY? FUNCTION (IF errorCapturingNot EXISTS)? - identifierReference LEFT_PAREN parameters=colDefinitionList? RIGHT_PAREN - (RETURNS (dataType | TABLE LEFT_PAREN returnParams=colTypeList RIGHT_PAREN))? - routineCharacteristics - RETURN (query | expression) #createUserDefinedFunction - | DROP TEMPORARY? FUNCTION (IF EXISTS)? identifierReference #dropFunction - | DECLARE (OR REPLACE)? variable? - identifierReference dataType? variableDefaultExpression? #createVariable - | DROP TEMPORARY variable (IF EXISTS)? identifierReference #dropVariable + | DROP TEMPORARY? FUNCTION (IF EXISTS)? identifierReference #dropFunction | EXPLAIN (LOGICAL | FORMATTED | EXTENDED | CODEGEN | COST)? - (statement|setResetStatement) #explain + statement #explain | SHOW TABLES ((FROM | IN) identifierReference)? (LIKE? pattern=stringLit)? #showTables | SHOW TABLE EXTENDED ((FROM | IN) ns=identifierReference)? @@ -270,51 +206,22 @@ statement | (MSCK)? REPAIR TABLE identifierReference (option=(ADD|DROP|SYNC) PARTITIONS)? #repairTable | op=(ADD | LIST) identifier .*? #manageResource - | CREATE INDEX (IF errorCapturingNot EXISTS)? identifier ON TABLE? - identifierReference (USING indexType=identifier)? - LEFT_PAREN columns=multipartIdentifierPropertyList RIGHT_PAREN - (OPTIONS options=propertyList)? #createIndex - | DROP INDEX (IF EXISTS)? identifier ON TABLE? identifierReference #dropIndex - | unsupportedHiveNativeCommands .*? #failNativeCommand - ; - -setResetStatement - : SET COLLATION collationName=identifier #setCollation - | SET ROLE .*? #failSetRole + | SET ROLE .*? #failNativeCommand | SET TIME ZONE interval #setTimeZone | SET TIME ZONE timezone #setTimeZone | SET TIME ZONE .*? #setTimeZone - | SET variable assignmentList #setVariable - | SET variable LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ - LEFT_PAREN query RIGHT_PAREN #setVariable | SET configKey EQ configValue #setQuotedConfiguration | SET configKey (EQ .*?)? #setConfiguration | SET .*? EQ configValue #setQuotedConfiguration | SET .*? #setConfiguration | RESET configKey #resetQuotedConfiguration | RESET .*? #resetConfiguration - ; - -executeImmediate - : EXECUTE IMMEDIATE queryParam=executeImmediateQueryParam (INTO targetVariable=multipartIdentifierList)? executeImmediateUsing? - ; - -executeImmediateUsing - : USING LEFT_PAREN params=namedExpressionSeq RIGHT_PAREN - | USING params=namedExpressionSeq - ; - -executeImmediateQueryParam - : stringLit - | multipartIdentifier - ; - -executeImmediateArgument - : (constant|multipartIdentifier) (AS name=errorCapturingIdentifier)? - ; - -executeImmediateArgumentSeq - : executeImmediateArgument (COMMA executeImmediateArgument)* + | CREATE INDEX (IF NOT EXISTS)? identifier ON TABLE? + identifierReference (USING indexType=identifier)? + LEFT_PAREN columns=multipartIdentifierPropertyList RIGHT_PAREN + (OPTIONS options=propertyList)? #createIndex + | DROP INDEX (IF EXISTS)? identifier ON TABLE? identifierReference #dropIndex + | unsupportedHiveNativeCommands .*? #failNativeCommand ; timezone @@ -378,17 +285,13 @@ unsupportedHiveNativeCommands ; createTableHeader - : CREATE TEMPORARY? EXTERNAL? TABLE (IF errorCapturingNot EXISTS)? identifierReference + : CREATE TEMPORARY? EXTERNAL? TABLE (IF NOT EXISTS)? identifierReference ; replaceTableHeader : (CREATE OR)? REPLACE TABLE identifierReference ; -clusterBySpec - : CLUSTER BY LEFT_PAREN multipartIdentifierList RIGHT_PAREN - ; - bucketSpec : CLUSTERED BY identifierList (SORTED BY orderedIdentifierList)? @@ -405,10 +308,6 @@ locationSpec : LOCATION stringLit ; -schemaBinding - : WITH SCHEMA (BINDING | COMPENSATION | EVOLUTION | TYPE EVOLUTION) - ; - commentSpec : COMMENT stringLit ; @@ -418,9 +317,9 @@ query ; insertInto - : INSERT OVERWRITE TABLE? identifierReference optionsClause? (partitionSpec (IF errorCapturingNot EXISTS)?)? ((BY NAME) | identifierList)? #insertOverwriteTable - | INSERT INTO TABLE? identifierReference optionsClause? partitionSpec? (IF errorCapturingNot EXISTS)? ((BY NAME) | identifierList)? #insertIntoTable - | INSERT INTO TABLE? identifierReference optionsClause? REPLACE whereClause #insertIntoReplaceWhere + : INSERT OVERWRITE TABLE? identifierReference (partitionSpec (IF NOT EXISTS)?)? ((BY NAME) | identifierList)? #insertOverwriteTable + | INSERT INTO TABLE? identifierReference partitionSpec? (IF NOT EXISTS)? ((BY NAME) | identifierList)? #insertIntoTable + | INSERT INTO TABLE? identifierReference REPLACE whereClause #insertIntoReplaceWhere | INSERT OVERWRITE LOCAL? DIRECTORY path=stringLit rowFormat? createFileFormat? #insertOverwriteHiveDir | INSERT OVERWRITE LOCAL? DIRECTORY (path=stringLit)? tableProvider (OPTIONS options=propertyList)? #insertOverwriteDir ; @@ -450,23 +349,16 @@ namespaces | SCHEMAS ; -variable - : VARIABLE - | VAR - ; - describeFuncName : identifierReference | stringLit | comparisonOperator | arithmeticOperator | predicateOperator - | shiftOperator - | BANG ; describeColName - : nameParts+=errorCapturingIdentifier (DOT nameParts+=errorCapturingIdentifier)* + : nameParts+=identifier (DOT nameParts+=identifier)* ; ctes @@ -485,7 +377,6 @@ createTableClauses :((OPTIONS options=expressionPropertyList) | (PARTITIONED BY partitioning=partitionFieldList) | skewSpec | - clusterBySpec | bucketSpec | rowFormat | createFileFormat | @@ -503,7 +394,7 @@ property ; propertyKey - : errorCapturingIdentifier (DOT errorCapturingIdentifier)* + : identifier (DOT identifier)* | stringLit ; @@ -553,7 +444,7 @@ dmlStatementNoWith | fromClause multiInsertQueryBody+ #multiInsertQuery | DELETE FROM identifierReference tableAlias whereClause? #deleteFromTable | UPDATE identifierReference tableAlias setClause whereClause? #updateTable - | MERGE (WITH SCHEMA EVOLUTION)? INTO target=identifierReference targetAlias=tableAlias + | MERGE INTO target=identifierReference targetAlias=tableAlias USING (source=identifierReference | LEFT_PAREN sourceQuery=query RIGHT_PAREN) sourceAlias=tableAlias ON mergeCondition=booleanExpression @@ -661,11 +552,11 @@ matchedClause : WHEN MATCHED (AND matchedCond=booleanExpression)? THEN matchedAction ; notMatchedClause - : WHEN errorCapturingNot MATCHED (BY TARGET)? (AND notMatchedCond=booleanExpression)? THEN notMatchedAction + : WHEN NOT MATCHED (BY TARGET)? (AND notMatchedCond=booleanExpression)? THEN notMatchedAction ; notMatchedBySourceClause - : WHEN errorCapturingNot MATCHED BY SOURCE (AND notMatchedBySourceCond=booleanExpression)? THEN notMatchedBySourceAction + : WHEN NOT MATCHED BY SOURCE (AND notMatchedBySourceCond=booleanExpression)? THEN notMatchedBySourceAction ; matchedAction @@ -685,10 +576,6 @@ notMatchedBySourceAction | UPDATE SET assignmentList ; -exceptClause - : EXCEPT LEFT_PAREN exceptCols=multipartIdentifierList RIGHT_PAREN - ; - assignmentList : assignment (COMMA assignment)* ; @@ -757,18 +644,18 @@ pivotClause ; pivotColumn - : identifiers+=errorCapturingIdentifier - | LEFT_PAREN identifiers+=errorCapturingIdentifier (COMMA identifiers+=errorCapturingIdentifier)* RIGHT_PAREN + : identifiers+=identifier + | LEFT_PAREN identifiers+=identifier (COMMA identifiers+=identifier)* RIGHT_PAREN ; pivotValue - : expression (AS? errorCapturingIdentifier)? + : expression (AS? identifier)? ; unpivotClause : UNPIVOT nullOperator=unpivotNullClause? LEFT_PAREN operator=unpivotOperator - RIGHT_PAREN (AS? errorCapturingIdentifier)? + RIGHT_PAREN (AS? identifier)? ; unpivotNullClause @@ -810,7 +697,7 @@ unpivotColumn ; unpivotAlias - : AS? errorCapturingIdentifier + : AS? identifier ; lateralView @@ -890,37 +777,21 @@ identifierComment relationPrimary : identifierReference temporalClause? - optionsClause? sample? tableAlias #tableName + sample? tableAlias #tableName | LEFT_PAREN query RIGHT_PAREN sample? tableAlias #aliasedQuery | LEFT_PAREN relation RIGHT_PAREN sample? tableAlias #aliasedRelation | inlineTable #inlineTableDefault2 | functionTable #tableValuedFunction ; -optionsClause - : WITH options=propertyList - ; - inlineTable : VALUES expression (COMMA expression)* tableAlias ; functionTableSubqueryArgument - : TABLE identifierReference tableArgumentPartitioning? - | TABLE LEFT_PAREN identifierReference RIGHT_PAREN tableArgumentPartitioning? - | TABLE LEFT_PAREN query RIGHT_PAREN tableArgumentPartitioning? - ; - -tableArgumentPartitioning - : ((WITH SINGLE PARTITION) - | ((PARTITION | DISTRIBUTE) BY - (((LEFT_PAREN partition+=expression (COMMA partition+=expression)* RIGHT_PAREN)) - | (expression (COMMA invalidMultiPartitionExpression=expression)+) - | partition+=expression))) - ((ORDER | SORT) BY - (((LEFT_PAREN sortItem (COMMA sortItem)* RIGHT_PAREN) - | (sortItem (COMMA invalidMultiSortItem=sortItem)+) - | sortItem)))? + : TABLE identifierReference + | TABLE LEFT_PAREN identifierReference RIGHT_PAREN + | TABLE LEFT_PAREN query RIGHT_PAREN ; functionTableNamedArgumentExpression @@ -1027,7 +898,7 @@ expressionSeq ; booleanExpression - : (NOT | BANG) booleanExpression #logicalNot + : NOT booleanExpression #logicalNot | EXISTS LEFT_PAREN query RIGHT_PAREN #exists | valueExpression predicate? #predicated | left=booleanExpression operator=AND right=booleanExpression #logicalBinary @@ -1035,20 +906,15 @@ booleanExpression ; predicate - : errorCapturingNot? kind=BETWEEN lower=valueExpression AND upper=valueExpression - | errorCapturingNot? kind=IN LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN - | errorCapturingNot? kind=IN LEFT_PAREN query RIGHT_PAREN - | errorCapturingNot? kind=RLIKE pattern=valueExpression - | errorCapturingNot? kind=(LIKE | ILIKE) quantifier=(ANY | SOME | ALL) (LEFT_PAREN RIGHT_PAREN | LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN) - | errorCapturingNot? kind=(LIKE | ILIKE) pattern=valueExpression (ESCAPE escapeChar=stringLit)? - | IS errorCapturingNot? kind=NULL - | IS errorCapturingNot? kind=(TRUE | FALSE | UNKNOWN) - | IS errorCapturingNot? kind=DISTINCT FROM right=valueExpression - ; - -errorCapturingNot - : NOT - | BANG + : NOT? kind=BETWEEN lower=valueExpression AND upper=valueExpression + | NOT? kind=IN LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN + | NOT? kind=IN LEFT_PAREN query RIGHT_PAREN + | NOT? kind=RLIKE pattern=valueExpression + | NOT? kind=(LIKE | ILIKE) quantifier=(ANY | SOME | ALL) (LEFT_PAREN RIGHT_PAREN | LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN) + | NOT? kind=(LIKE | ILIKE) pattern=valueExpression (ESCAPE escapeChar=stringLit)? + | IS NOT? kind=NULL + | IS NOT? kind=(TRUE | FALSE | UNKNOWN) + | IS NOT? kind=DISTINCT FROM right=valueExpression ; valueExpression @@ -1056,19 +922,12 @@ valueExpression | operator=(MINUS | PLUS | TILDE) valueExpression #arithmeticUnary | left=valueExpression operator=(ASTERISK | SLASH | PERCENT | DIV) right=valueExpression #arithmeticBinary | left=valueExpression operator=(PLUS | MINUS | CONCAT_PIPE) right=valueExpression #arithmeticBinary - | left=valueExpression shiftOperator right=valueExpression #shiftExpression | left=valueExpression operator=AMPERSAND right=valueExpression #arithmeticBinary | left=valueExpression operator=HAT right=valueExpression #arithmeticBinary | left=valueExpression operator=PIPE right=valueExpression #arithmeticBinary | left=valueExpression comparisonOperator right=valueExpression #comparison ; -shiftOperator - : SHIFT_LEFT - | SHIFT_RIGHT - | SHIFT_RIGHT_UNSIGNED - ; - datetimeUnit : YEAR | QUARTER | MONTH | WEEK | DAY | DAYOFYEAR @@ -1076,27 +935,24 @@ datetimeUnit ; primaryExpression - : name=(CURRENT_DATE | CURRENT_TIMESTAMP | CURRENT_USER | USER | SESSION_USER) #currentLike + : name=(CURRENT_DATE | CURRENT_TIMESTAMP | CURRENT_USER | USER) #currentLike | name=(TIMESTAMPADD | DATEADD | DATE_ADD) LEFT_PAREN (unit=datetimeUnit | invalidUnit=stringLit) COMMA unitsAmount=valueExpression COMMA timestamp=valueExpression RIGHT_PAREN #timestampadd - | name=(TIMESTAMPDIFF | DATEDIFF | DATE_DIFF | TIMEDIFF) LEFT_PAREN (unit=datetimeUnit | invalidUnit=stringLit) COMMA startTimestamp=valueExpression COMMA endTimestamp=valueExpression RIGHT_PAREN #timestampdiff + | name=(TIMESTAMPDIFF | DATEDIFF | DATE_DIFF) LEFT_PAREN (unit=datetimeUnit | invalidUnit=stringLit) COMMA startTimestamp=valueExpression COMMA endTimestamp=valueExpression RIGHT_PAREN #timestampdiff | CASE whenClause+ (ELSE elseExpression=expression)? END #searchedCase | CASE value=expression whenClause+ (ELSE elseExpression=expression)? END #simpleCase | name=(CAST | TRY_CAST) LEFT_PAREN expression AS dataType RIGHT_PAREN #cast - | primaryExpression collateClause #collate - | primaryExpression DOUBLE_COLON dataType #castByColon | STRUCT LEFT_PAREN (argument+=namedExpression (COMMA argument+=namedExpression)*)? RIGHT_PAREN #struct | FIRST LEFT_PAREN expression (IGNORE NULLS)? RIGHT_PAREN #first | ANY_VALUE LEFT_PAREN expression (IGNORE NULLS)? RIGHT_PAREN #any_value | LAST LEFT_PAREN expression (IGNORE NULLS)? RIGHT_PAREN #last | POSITION LEFT_PAREN substr=valueExpression IN str=valueExpression RIGHT_PAREN #position | constant #constantDefault - | ASTERISK exceptClause? #star - | qualifiedName DOT ASTERISK exceptClause? #star + | ASTERISK #star + | qualifiedName DOT ASTERISK #star | LEFT_PAREN namedExpression (COMMA namedExpression)+ RIGHT_PAREN #rowConstructor | LEFT_PAREN query RIGHT_PAREN #subqueryExpression | functionName LEFT_PAREN (setQuantifier? argument+=functionArgument (COMMA argument+=functionArgument)*)? RIGHT_PAREN - (WITHIN GROUP LEFT_PAREN ORDER BY sortItem (COMMA sortItem)* RIGHT_PAREN)? (FILTER LEFT_PAREN WHERE where=booleanExpression RIGHT_PAREN)? (nullsOption=(IGNORE | RESPECT) NULLS)? ( OVER windowSpec)? #functionCall | identifier ARROW expression #lambda @@ -1112,6 +968,9 @@ primaryExpression FROM srcStr=valueExpression RIGHT_PAREN #trim | OVERLAY LEFT_PAREN input=valueExpression PLACING replace=valueExpression FROM position=valueExpression (FOR length=valueExpression)? RIGHT_PAREN #overlay + | name=(PERCENTILE_CONT | PERCENTILE_DISC) LEFT_PAREN percentage=valueExpression RIGHT_PAREN + WITHIN GROUP LEFT_PAREN ORDER BY sortItem RIGHT_PAREN + (FILTER LEFT_PAREN WHERE where=booleanExpression RIGHT_PAREN)? ( OVER windowSpec)? #percentile ; literalType @@ -1188,10 +1047,6 @@ colPosition : position=FIRST | position=AFTER afterCol=errorCapturingIdentifier ; -collateClause - : COLLATE collationName=identifier - ; - type : BOOLEAN | TINYINT | BYTE @@ -1202,14 +1057,13 @@ type | DOUBLE | DATE | TIMESTAMP | TIMESTAMP_NTZ | TIMESTAMP_LTZ - | STRING collateClause? + | STRING | CHARACTER | CHAR | VARCHAR | BINARY | DECIMAL | DEC | NUMERIC | VOID | INTERVAL - | VARIANT | ARRAY | STRUCT | MAP | unsupportedType=identifier ; @@ -1234,7 +1088,7 @@ qualifiedColTypeWithPosition ; colDefinitionDescriptorWithPosition - : errorCapturingNot NULL + : NOT NULL | defaultExpression | commentSpec | colPosition @@ -1244,28 +1098,24 @@ defaultExpression : DEFAULT expression ; -variableDefaultExpression - : (DEFAULT | EQ) expression - ; - colTypeList : colType (COMMA colType)* ; colType - : colName=errorCapturingIdentifier dataType (errorCapturingNot NULL)? commentSpec? + : colName=errorCapturingIdentifier dataType (NOT NULL)? commentSpec? ; -colDefinitionList - : colDefinition (COMMA colDefinition)* +createOrReplaceTableColTypeList + : createOrReplaceTableColType (COMMA createOrReplaceTableColType)* ; -colDefinition +createOrReplaceTableColType : colName=errorCapturingIdentifier dataType colDefinitionOption* ; colDefinitionOption - : errorCapturingNot NULL + : NOT NULL | defaultExpression | generationExpression | commentSpec @@ -1280,49 +1130,9 @@ complexColTypeList ; complexColType - : errorCapturingIdentifier COLON? dataType (errorCapturingNot NULL)? commentSpec? - ; - -routineCharacteristics - : (routineLanguage - | specificName - | deterministic - | sqlDataAccess - | nullCall - | commentSpec - | rightsClause)* - ; - -routineLanguage - : LANGUAGE (SQL | IDENTIFIER) - ; - -specificName - : SPECIFIC specific=errorCapturingIdentifier - ; - -deterministic - : DETERMINISTIC - | errorCapturingNot DETERMINISTIC + : identifier COLON? dataType (NOT NULL)? commentSpec? ; -sqlDataAccess - : access=NO SQL - | access=CONTAINS SQL - | access=READS SQL DATA - | access=MODIFIES SQL DATA - ; - -nullCall - : RETURNS NULL ON NULL INPUT - | CALLED ON NULL INPUT - ; - -rightsClause - : SQL SECURITY INVOKER - | SQL SECURITY DEFINER - ; - whenClause : WHEN condition=expression THEN result=expression ; @@ -1427,7 +1237,7 @@ alterColumnAction : TYPE dataType | commentSpec | colPosition - | setOrDrop=(SET | DROP) errorCapturingNot NULL + | setOrDrop=(SET | DROP) NOT NULL | SET defaultExpression | dropDefault=DROP DEFAULT ; @@ -1470,19 +1280,16 @@ ansiNonReserved | ARRAY | ASC | AT - | BEGIN | BETWEEN | BIGINT | BINARY | BINARY_HEX - | BINDING | BOOLEAN | BUCKET | BUCKETS | BY | BYTE | CACHE - | CALLED | CASCADE | CATALOG | CATALOGS @@ -1499,10 +1306,8 @@ ansiNonReserved | COMMIT | COMPACT | COMPACTIONS - | COMPENSATION | COMPUTE | CONCATENATE - | CONTAINS | COST | CUBE | CURRENT @@ -1520,25 +1325,20 @@ ansiNonReserved | DBPROPERTIES | DEC | DECIMAL - | DECLARE | DEFAULT | DEFINED - | DEFINER | DELETE | DELIMITED | DESC | DESCRIBE - | DETERMINISTIC | DFS | DIRECTORIES | DIRECTORY | DISTRIBUTE | DIV - | DO | DOUBLE | DROP | ESCAPED - | EVOLUTION | EXCHANGE | EXCLUDE | EXISTS @@ -1564,22 +1364,18 @@ ansiNonReserved | IDENTIFIER_KW | IF | IGNORE - | IMMEDIATE | IMPORT | INCLUDE | INDEX | INDEXES | INPATH - | INPUT | INPUTFORMAT | INSERT | INT | INTEGER | INTERVAL - | INVOKER | ITEMS | KEYS - | LANGUAGE | LAST | LAZY | LIKE @@ -1604,7 +1400,6 @@ ansiNonReserved | MILLISECONDS | MINUTE | MINUTES - | MODIFIES | MONTH | MONTHS | MSCK @@ -1614,7 +1409,6 @@ ansiNonReserved | NANOSECOND | NANOSECONDS | NO - | NONE | NULLS | NUMERIC | OF @@ -1639,7 +1433,6 @@ ansiNonReserved | QUARTER | QUERY | RANGE - | READS | REAL | RECORDREADER | RECORDWRITER @@ -1653,8 +1446,6 @@ ansiNonReserved | RESET | RESPECT | RESTRICT - | RETURN - | RETURNS | REVOKE | RLIKE | ROLE @@ -1667,7 +1458,6 @@ ansiNonReserved | SCHEMAS | SECOND | SECONDS - | SECURITY | SEMI | SEPARATED | SERDE @@ -1677,13 +1467,11 @@ ansiNonReserved | SETS | SHORT | SHOW - | SINGLE | SKEWED | SMALLINT | SORT | SORTED | SOURCE - | SPECIFIC | START | STATISTICS | STORED @@ -1701,7 +1489,6 @@ ansiNonReserved | TBLPROPERTIES | TEMPORARY | TERMINATED - | TIMEDIFF | TIMESTAMP | TIMESTAMP_LTZ | TIMESTAMP_NTZ @@ -1727,16 +1514,12 @@ ansiNonReserved | USE | VALUES | VARCHAR - | VAR - | VARIABLE - | VARIANT | VERSION | VIEW | VIEWS | VOID | WEEK | WEEKS - | WHILE | WINDOW | YEAR | YEARS @@ -1789,12 +1572,10 @@ nonReserved | ASC | AT | AUTHORIZATION - | BEGIN | BETWEEN | BIGINT | BINARY | BINARY_HEX - | BINDING | BOOLEAN | BOTH | BUCKET @@ -1802,7 +1583,6 @@ nonReserved | BY | BYTE | CACHE - | CALLED | CASCADE | CASE | CAST @@ -1817,7 +1597,6 @@ nonReserved | CLUSTERED | CODEGEN | COLLATE - | COLLATION | COLLECTION | COLUMN | COLUMNS @@ -1825,11 +1604,9 @@ nonReserved | COMMIT | COMPACT | COMPACTIONS - | COMPENSATION | COMPUTE | CONCATENATE | CONSTRAINT - | CONTAINS | COST | CREATE | CUBE @@ -1852,32 +1629,26 @@ nonReserved | DBPROPERTIES | DEC | DECIMAL - | DECLARE | DEFAULT | DEFINED - | DEFINER | DELETE | DELIMITED | DESC | DESCRIBE - | DETERMINISTIC | DFS | DIRECTORIES | DIRECTORY | DISTINCT | DISTRIBUTE | DIV - | DO | DOUBLE | DROP | ELSE | END | ESCAPE | ESCAPED - | EVOLUTION | EXCHANGE | EXCLUDE - | EXECUTE | EXISTS | EXPLAIN | EXPORT @@ -1910,25 +1681,21 @@ nonReserved | IDENTIFIER_KW | IF | IGNORE - | IMMEDIATE | IMPORT | IN | INCLUDE | INDEX | INDEXES | INPATH - | INPUT | INPUTFORMAT | INSERT | INT | INTEGER | INTERVAL | INTO - | INVOKER | IS | ITEMS | KEYS - | LANGUAGE | LAST | LAZY | LEADING @@ -1955,7 +1722,6 @@ nonReserved | MILLISECONDS | MINUTE | MINUTES - | MODIFIES | MONTH | MONTHS | MSCK @@ -1965,7 +1731,6 @@ nonReserved | NANOSECOND | NANOSECONDS | NO - | NONE | NOT | NULL | NULLS @@ -1987,6 +1752,8 @@ nonReserved | PARTITION | PARTITIONED | PARTITIONS + | PERCENTILE_CONT + | PERCENTILE_DISC | PERCENTLIT | PIVOT | PLACING @@ -1999,7 +1766,6 @@ nonReserved | QUARTER | QUERY | RANGE - | READS | REAL | RECORDREADER | RECORDWRITER @@ -2014,8 +1780,6 @@ nonReserved | RESET | RESPECT | RESTRICT - | RETURN - | RETURNS | REVOKE | RLIKE | ROLE @@ -2028,7 +1792,6 @@ nonReserved | SCHEMAS | SECOND | SECONDS - | SECURITY | SELECT | SEPARATED | SERDE @@ -2038,15 +1801,12 @@ nonReserved | SETS | SHORT | SHOW - | SINGLE | SKEWED | SMALLINT | SOME | SORT | SORTED | SOURCE - | SPECIFIC - | SQL | START | STATISTICS | STORED @@ -2067,7 +1827,6 @@ nonReserved | TERMINATED | THEN | TIME - | TIMEDIFF | TIMESTAMP | TIMESTAMP_LTZ | TIMESTAMP_NTZ @@ -2098,16 +1857,12 @@ nonReserved | USER | VALUES | VARCHAR - | VAR - | VARIABLE - | VARIANT | VERSION | VIEW | VIEWS | VOID | WEEK | WEEKS - | WHILE | WHEN | WHERE | WINDOW From 89f8191867d5cca5fa0af17e721b622c7f27af86 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:09:09 -0700 Subject: [PATCH 10/32] [Backport 2.x] Push down limit through eval (#2927) (cherry picked from commit 4a735ea9bca6313a06616a99a944b8e512baeb66) Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../optimizer/LogicalPlanOptimizer.java | 2 + .../planner/optimizer/pattern/Patterns.java | 5 ++ .../planner/optimizer/rule/EvalPushDown.java | 82 +++++++++++++++++++ .../optimizer/LogicalPlanOptimizerTest.java | 23 ++++++ .../org/opensearch/sql/ppl/ExplainIT.java | 13 +++ .../ppl/explain_limit_push.json | 27 ++++++ 6 files changed, 152 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/planner/optimizer/rule/EvalPushDown.java create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java index 5c115f0db8..e805b0dea5 100644 --- a/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.stream.Collectors; import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.optimizer.rule.EvalPushDown; import org.opensearch.sql.planner.optimizer.rule.MergeFilterAndFilter; import org.opensearch.sql.planner.optimizer.rule.PushFilterUnderSort; import org.opensearch.sql.planner.optimizer.rule.read.CreateTableScanBuilder; @@ -46,6 +47,7 @@ public static LogicalPlanOptimizer create() { */ new MergeFilterAndFilter(), new PushFilterUnderSort(), + EvalPushDown.PUSH_DOWN_LIMIT, /* * Phase 2: Transformations that rely on data source push down capability */ diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java index ee4e9a20cc..ef2607e018 100644 --- a/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java @@ -12,6 +12,7 @@ import java.util.Optional; import lombok.experimental.UtilityClass; import org.opensearch.sql.planner.logical.LogicalAggregation; +import org.opensearch.sql.planner.logical.LogicalEval; import org.opensearch.sql.planner.logical.LogicalFilter; import org.opensearch.sql.planner.logical.LogicalHighlight; import org.opensearch.sql.planner.logical.LogicalLimit; @@ -63,6 +64,10 @@ public static Pattern project(Pattern return Pattern.typeOf(LogicalProject.class).with(source(pattern)); } + public static Pattern evalCapture() { + return Pattern.typeOf(LogicalEval.class).capturedAs(Capture.newCapture()); + } + /** Pattern for {@link TableScanBuilder} and capture it meanwhile. */ public static Pattern scanBuilder() { return Pattern.typeOf(TableScanBuilder.class).capturedAs(Capture.newCapture()); diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/EvalPushDown.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/EvalPushDown.java new file mode 100644 index 0000000000..17eaed0e8c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/EvalPushDown.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.optimizer.rule; + +import static org.opensearch.sql.planner.optimizer.pattern.Patterns.evalCapture; +import static org.opensearch.sql.planner.optimizer.pattern.Patterns.limit; +import static org.opensearch.sql.planner.optimizer.rule.EvalPushDown.EvalPushDownBuilder.match; + +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.matching.pattern.CapturePattern; +import com.facebook.presto.matching.pattern.WithPattern; +import java.util.List; +import java.util.function.BiFunction; +import lombok.Getter; +import lombok.experimental.Accessors; +import org.opensearch.sql.planner.logical.LogicalEval; +import org.opensearch.sql.planner.logical.LogicalLimit; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.optimizer.Rule; + +/** + * Rule template for all rules related to push down logical plans under eval, so these plans can + * avoid blocking by eval and may have chances to be pushed down into table scan by rules in {@link + * org.opensearch.sql.planner.optimizer.rule.read.TableScanPushDown}. + */ +public class EvalPushDown implements Rule { + + // TODO: Add more rules to push down sort and project + /** Push down optimize rule for limit operator. Transform `limit -> eval` to `eval -> limit` */ + public static final Rule PUSH_DOWN_LIMIT = + match(limit(evalCapture())) + .apply( + (limit, logicalEval) -> { + List child = logicalEval.getChild(); + limit.replaceChildPlans(child); + logicalEval.replaceChildPlans(List.of(limit)); + return logicalEval; + }); + + private final Capture capture; + + @Accessors(fluent = true) + @Getter + private final Pattern pattern; + + private final BiFunction pushDownFunction; + + @SuppressWarnings("unchecked") + public EvalPushDown( + WithPattern pattern, BiFunction pushDownFunction) { + this.pattern = pattern; + this.capture = ((CapturePattern) pattern.getPattern()).capture(); + this.pushDownFunction = pushDownFunction; + } + + @Override + public LogicalPlan apply(T plan, Captures captures) { + LogicalEval logicalEval = captures.get(capture); + return pushDownFunction.apply(plan, logicalEval); + } + + static class EvalPushDownBuilder { + + private WithPattern pattern; + + public static EvalPushDown.EvalPushDownBuilder match( + Pattern pattern) { + EvalPushDown.EvalPushDownBuilder builder = new EvalPushDown.EvalPushDownBuilder<>(); + builder.pattern = (WithPattern) pattern; + return builder; + } + + public EvalPushDown apply(BiFunction pushDownFunction) { + return new EvalPushDown<>(pattern, pushDownFunction); + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java b/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java index c25e415cfa..20996503b4 100644 --- a/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.data.model.ExprValueUtils.longValue; import static org.opensearch.sql.data.type.ExprCoreType.*; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.aggregation; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.eval; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.filter; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.highlight; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.limit; @@ -43,6 +44,7 @@ import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.planner.logical.LogicalPaginate; @@ -345,6 +347,27 @@ void table_scan_builder_support_offset_push_down_can_apply_its_rule() { assertEquals(project(tableScanBuilder), optimized); } + /** Limit - Eval --> Eval - Limit. */ + @Test + void push_limit_under_eval() { + Pair evalExpr = + Pair.of(DSL.ref("name1", STRING), DSL.ref("name", STRING)); + assertEquals( + eval(limit(tableScanBuilder, 10, 5), evalExpr), + optimize(limit(eval(relation("schema", table), evalExpr), 10, 5))); + } + + /** Limit - Eval - Scan --> Eval - Scan. */ + @Test + void push_limit_through_eval_into_scan() { + when(tableScanBuilder.pushDownLimit(any())).thenReturn(true); + Pair evalExpr = + Pair.of(DSL.ref("name1", STRING), DSL.ref("name", STRING)); + assertEquals( + eval(tableScanBuilder, evalExpr), + optimize(limit(eval(relation("schema", table), evalExpr), 10, 5))); + } + private LogicalPlan optimize(LogicalPlan plan) { final LogicalPlanOptimizer optimizer = LogicalPlanOptimizer.create(); return optimizer.optimize(plan); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index fce975ef92..c6b21e1605 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -76,6 +76,19 @@ public void testSortPushDownExplain() throws Exception { + "| fields age")); } + @Test + public void testLimitPushDownExplain() throws Exception { + String expected = loadFromFile("expectedOutput/ppl/explain_limit_push.json"); + + assertJsonEquals( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account" + + "| eval ageMinus = age - 30 " + + "| head 5 " + + "| fields ageMinus")); + } + String loadFromFile(String filename) throws Exception { URI uri = Resources.getResource(filename).toURI(); return new String(Files.readAllBytes(Paths.get(uri))); diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json new file mode 100644 index 0000000000..51a627ea4d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json @@ -0,0 +1,27 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[ageMinus]" + }, + "children": [ + { + "name": "EvalOperator", + "description": { + "expressions": { + "ageMinus": "-(age, 30)" + } + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, searchDone=false)" + }, + "children": [] + } + ] + } + ] + } +} From 79f9be0be24191c2bdef1fc39d08c759f4ebcd7f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:09:17 -0700 Subject: [PATCH 11/32] [Backport 2.x] Add job scheduler plugin to test cluster (#3047) (cherry picked from commit c1e623d6d3464056a4b16ae7e51dca218d73a5b6) Signed-off-by: Louis Chu Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- plugin/build.gradle | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/plugin/build.gradle b/plugin/build.gradle index 17b0ae8573..ebfceadb0f 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -1,3 +1,5 @@ +import java.util.concurrent.Callable + /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 @@ -113,6 +115,11 @@ configurations.all { resolutionStrategy.force "org.apache.httpcomponents:httpclient:4.5.14" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" } + +configurations { + zipArchive +} + compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } @@ -143,6 +150,8 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-core', version: "${versions.mockito}" testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${versions.mockito}" testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' + + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" } test { @@ -252,7 +261,24 @@ afterEvaluate { } } +def getJobSchedulerPlugin() { + provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile + } + } + } + }) +} + testClusters.integTest { + plugin(getJobSchedulerPlugin()) plugin(project.tasks.bundlePlugin.archiveFile) testDistribution = "ARCHIVE" From 0ca3f85fcea512ffd30a38e5bcd3b991e47a64c5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 14:54:42 -0700 Subject: [PATCH 12/32] [Backport 2.x] Unify internal OpenSearch exceptions with returned status code (#3072) (cherry picked from commit 4c44f56f3eed5926430acf2c6a12151cbf182162) Signed-off-by: Simeon Widdis Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../src/test/java/org/opensearch/sql/legacy/CursorIT.java | 2 +- .../sql/legacy/executor/format/OpenSearchErrorMessage.java | 4 ++-- .../sql/opensearch/response/error/OpenSearchErrorMessage.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index b6a18d6de9..1fc532c1f8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -376,7 +376,7 @@ public void invalidCursorIdNotDecodable() throws IOException { JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); assertThat(resp.getInt("status"), equalTo(400)); - assertThat(resp.query("/error/type"), equalTo("illegal_argument_exception")); + assertThat(resp.query("/error/type"), equalTo("IllegalArgumentException")); } /** diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/OpenSearchErrorMessage.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/OpenSearchErrorMessage.java index 8117d241b1..09c09919ec 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/OpenSearchErrorMessage.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/OpenSearchErrorMessage.java @@ -12,8 +12,8 @@ public class OpenSearchErrorMessage extends ErrorMessage { - OpenSearchErrorMessage(OpenSearchException exception, int status) { - super(exception, status); + OpenSearchErrorMessage(OpenSearchException exception, int defaultStatus) { + super(exception, exception.status() != null ? exception.status().getStatus() : defaultStatus); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessage.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessage.java index 87a374d353..a712ceaedf 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessage.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessage.java @@ -13,8 +13,8 @@ /** OpenSearch Error Message. */ public class OpenSearchErrorMessage extends ErrorMessage { - OpenSearchErrorMessage(OpenSearchException exception, int status) { - super(exception, status); + OpenSearchErrorMessage(OpenSearchException exception, int defaultStatus) { + super(exception, exception.status() != null ? exception.status().getStatus() : defaultStatus); } @Override From 66f5ae819dc3e68130d824ade95dc20699d6523e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:01:57 -0700 Subject: [PATCH 13/32] [Backport 2.x] Updating JobExecutionResponseReader interface to add RequestContext (#3074) * Updating JobExecutionResponseReader interface to add RequestContext * Updating JobExecutionResponseReader interface to add RequestContext --------- (cherry picked from commit 7666a44b7e07bd673bdd35d94a33f87cd98e3905) Signed-off-by: AMIT YADAV Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] Co-authored-by: AMIT YADAV --- .../spark/dispatcher/BatchQueryHandler.java | 4 +-- .../sql/spark/dispatcher/IndexDMLHandler.java | 2 +- .../dispatcher/InteractiveQueryHandler.java | 2 +- .../response/JobExecutionResponseReader.java | 14 +++++--- .../asyncquery/AsyncQueryCoreIntegTest.java | 19 ++++++++-- .../dispatcher/SparkQueryDispatcherTest.java | 35 +++++++++++++++---- .../OpenSearchJobExecutionResponseReader.java | 13 +++++-- .../AsyncQueryGetResultSpecTest.java | 18 +++++++--- ...nSearchJobExecutionResponseReaderTest.java | 28 +++++++++++---- 9 files changed, 105 insertions(+), 30 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java index 5a775aa243..4396b45898 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java @@ -47,8 +47,8 @@ protected JSONObject getResponseFromResultIndex( AsyncQueryRequestContext asyncQueryRequestContext) { // either empty json when the result is not available or data with status // Fetch from Result Index - return jobExecutionResponseReader.getResultWithJobId( - asyncQueryJobMetadata.getJobId(), asyncQueryJobMetadata.getResultIndex()); + return jobExecutionResponseReader.getResultFromResultIndex( + asyncQueryJobMetadata, asyncQueryRequestContext); } @Override diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java index fe848593a7..71b20b4311 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java @@ -169,7 +169,7 @@ protected JSONObject getResponseFromResultIndex( AsyncQueryRequestContext asyncQueryRequestContext) { String queryId = asyncQueryJobMetadata.getQueryId(); return jobExecutionResponseReader.getResultWithQueryId( - queryId, asyncQueryJobMetadata.getResultIndex()); + queryId, asyncQueryJobMetadata.getResultIndex(), asyncQueryRequestContext); } @Override diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java index 75912f3a7c..1eaad1ca9d 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java @@ -56,7 +56,7 @@ protected JSONObject getResponseFromResultIndex( AsyncQueryRequestContext asyncQueryRequestContext) { String queryId = asyncQueryJobMetadata.getQueryId(); return jobExecutionResponseReader.getResultWithQueryId( - queryId, asyncQueryJobMetadata.getResultIndex()); + queryId, asyncQueryJobMetadata.getResultIndex(), asyncQueryRequestContext); } @Override diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java b/async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java index e3184b7326..237ce9c7f6 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java @@ -6,17 +6,22 @@ package org.opensearch.sql.spark.response; import org.json.JSONObject; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; /** Interface for reading job execution result */ public interface JobExecutionResponseReader { /** * Retrieves the job execution result based on the job ID. * - * @param jobId The job ID. - * @param resultLocation The location identifier where the result is stored (optional). + * @param asyncQueryJobMetadata metadata will have jobId and resultLocation and other required + * params. + * @param asyncQueryRequestContext request context passed to AsyncQueryExecutorService * @return A JSONObject containing the result data. */ - JSONObject getResultWithJobId(String jobId, String resultLocation); + JSONObject getResultFromResultIndex( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext); /** * Retrieves the job execution result based on the query ID. @@ -25,5 +30,6 @@ public interface JobExecutionResponseReader { * @param resultLocation The location identifier where the result is stored (optional). * @return A JSONObject containing the result data. */ - JSONObject getResultWithQueryId(String queryId, String resultLocation); + JSONObject getResultWithQueryId( + String queryId, String resultLocation, AsyncQueryRequestContext asyncQueryRequestContext); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 57ad4ecf42..d4a6b544c4 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -452,7 +452,8 @@ public void getResultOfInteractiveQuery() { .sessionId(SESSION_ID) .resultIndex(RESULT_INDEX)); JSONObject result = getValidExecutionResponse(); - when(jobExecutionResponseReader.getResultWithQueryId(QUERY_ID, RESULT_INDEX)) + when(jobExecutionResponseReader.getResultWithQueryId( + QUERY_ID, RESULT_INDEX, asyncQueryRequestContext)) .thenReturn(result); AsyncQueryExecutionResponse response = @@ -471,7 +472,8 @@ public void getResultOfIndexDMLQuery() { .jobId(DROP_INDEX_JOB_ID) .resultIndex(RESULT_INDEX)); JSONObject result = getValidExecutionResponse(); - when(jobExecutionResponseReader.getResultWithQueryId(QUERY_ID, RESULT_INDEX)) + when(jobExecutionResponseReader.getResultWithQueryId( + QUERY_ID, RESULT_INDEX, asyncQueryRequestContext)) .thenReturn(result); AsyncQueryExecutionResponse response = @@ -491,7 +493,18 @@ public void getResultOfRefreshQuery() { .jobType(JobType.BATCH) .resultIndex(RESULT_INDEX)); JSONObject result = getValidExecutionResponse(); - when(jobExecutionResponseReader.getResultWithJobId(JOB_ID, RESULT_INDEX)).thenReturn(result); + when(jobExecutionResponseReader.getResultFromResultIndex( + AsyncQueryJobMetadata.builder() + .applicationId(APPLICATION_ID) + .queryId(QUERY_ID) + .jobId(JOB_ID) + .datasourceName(DATASOURCE_NAME) + .resultIndex(RESULT_INDEX) + .jobType(JobType.BATCH) + .metadata(ImmutableMap.of()) + .build(), + asyncQueryRequestContext)) + .thenReturn(result); AsyncQueryExecutionResponse response = asyncQueryExecutorService.getAsyncQueryResults(QUERY_ID, asyncQueryRequestContext); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 1a38b6977f..343dc3040a 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -765,7 +765,17 @@ void testGetQueryResponse() { when(emrServerlessClient.getJobRunResult(EMRS_APPLICATION_ID, EMR_JOB_ID)) .thenReturn(new GetJobRunResult().withJobRun(new JobRun().withState(JobRunState.PENDING))); // simulate result index is not created yet - when(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)) + when(jobExecutionResponseReader.getResultFromResultIndex( + AsyncQueryJobMetadata.builder() + .jobId(EMR_JOB_ID) + .queryId(QUERY_ID) + .applicationId(EMRS_APPLICATION_ID) + .jobId(EMR_JOB_ID) + .jobType(JobType.INTERACTIVE) + .datasourceName(MY_GLUE) + .metadata(ImmutableMap.of()) + .build(), + asyncQueryRequestContext)) .thenReturn(new JSONObject()); JSONObject result = @@ -782,7 +792,7 @@ void testGetQueryResponseWithSession() { doReturn(StatementState.WAITING).when(statement).getStatementState(); doReturn(new JSONObject()) .when(jobExecutionResponseReader) - .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any()); + .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any(), eq(asyncQueryRequestContext)); JSONObject result = sparkQueryDispatcher.getQueryResponse( @@ -798,7 +808,7 @@ void testGetQueryResponseWithInvalidSession() { doReturn(Optional.empty()).when(sessionManager).getSession(MOCK_SESSION_ID, MY_GLUE); doReturn(new JSONObject()) .when(jobExecutionResponseReader) - .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any()); + .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any(), eq(asyncQueryRequestContext)); IllegalArgumentException exception = Assertions.assertThrows( @@ -818,7 +828,7 @@ void testGetQueryResponseWithStatementNotExist() { doReturn(Optional.empty()).when(session).get(any(), eq(asyncQueryRequestContext)); doReturn(new JSONObject()) .when(jobExecutionResponseReader) - .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any()); + .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any(), eq(asyncQueryRequestContext)); IllegalArgumentException exception = Assertions.assertThrows( @@ -840,12 +850,25 @@ void testGetQueryResponseWithSuccess() { resultMap.put(STATUS_FIELD, "SUCCESS"); resultMap.put(ERROR_FIELD, ""); queryResult.put(DATA_FIELD, resultMap); - when(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)).thenReturn(queryResult); + AsyncQueryJobMetadata asyncQueryJobMetadata = + AsyncQueryJobMetadata.builder() + .queryId(QUERY_ID) + .applicationId(EMRS_APPLICATION_ID) + .jobId(EMR_JOB_ID) + .jobType(JobType.INTERACTIVE) + .datasourceName(MY_GLUE) + .metadata(ImmutableMap.of()) + .jobId(EMR_JOB_ID) + .build(); + when(jobExecutionResponseReader.getResultFromResultIndex( + asyncQueryJobMetadata, asyncQueryRequestContext)) + .thenReturn(queryResult); JSONObject result = sparkQueryDispatcher.getQueryResponse(asyncQueryJobMetadata(), asyncQueryRequestContext); - verify(jobExecutionResponseReader, times(1)).getResultWithJobId(EMR_JOB_ID, null); + verify(jobExecutionResponseReader, times(1)) + .getResultFromResultIndex(asyncQueryJobMetadata, asyncQueryRequestContext); assertEquals( new HashSet<>(Arrays.asList(DATA_FIELD, STATUS_FIELD, ERROR_FIELD)), result.keySet()); JSONObject dataJson = new JSONObject(); diff --git a/async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java b/async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java index 10113ece8d..c969a3a6dc 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java @@ -21,6 +21,8 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; /** JobExecutionResponseReader implementation for reading response from OpenSearch index. */ public class OpenSearchJobExecutionResponseReader implements JobExecutionResponseReader { @@ -32,12 +34,17 @@ public OpenSearchJobExecutionResponseReader(Client client) { } @Override - public JSONObject getResultWithJobId(String jobId, String resultLocation) { - return searchInSparkIndex(QueryBuilders.termQuery(JOB_ID_FIELD, jobId), resultLocation); + public JSONObject getResultFromResultIndex( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { + return searchInSparkIndex( + QueryBuilders.termQuery(JOB_ID_FIELD, asyncQueryJobMetadata.getJobId()), + asyncQueryJobMetadata.getResultIndex()); } @Override - public JSONObject getResultWithQueryId(String queryId, String resultLocation) { + public JSONObject getResultWithQueryId( + String queryId, String resultLocation, AsyncQueryRequestContext asyncQueryRequestContext) { return searchInSparkIndex(QueryBuilders.termQuery("queryId", queryId), resultLocation); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java index 7ccbad969d..ef98e955f6 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java @@ -24,6 +24,7 @@ import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; import org.opensearch.sql.protocol.response.format.ResponseFormatter; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.asyncquery.model.MockFlintSparkJob; import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; @@ -428,12 +429,21 @@ private class AssertionHelper { */ new JobExecutionResponseReader() { @Override - public JSONObject getResultWithJobId(String jobId, String resultIndex) { - return interaction.interact(new InteractionStep(emrClient, jobId, resultIndex)); + public JSONObject getResultFromResultIndex( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { + return interaction.interact( + new InteractionStep( + emrClient, + asyncQueryJobMetadata.getJobId(), + asyncQueryJobMetadata.getResultIndex())); } @Override - public JSONObject getResultWithQueryId(String queryId, String resultIndex) { + public JSONObject getResultWithQueryId( + String queryId, + String resultIndex, + AsyncQueryRequestContext asyncQueryRequestContext) { return interaction.interact(new InteractionStep(emrClient, queryId, resultIndex)); } }); @@ -501,7 +511,7 @@ private InteractionStep(LocalEMRSClient emrClient, String queryId, String result /** Simulate PPL plugin search query_execution_result */ JSONObject pluginSearchQueryResult() { return new OpenSearchJobExecutionResponseReader(client) - .getResultWithQueryId(queryId, resultIndex); + .getResultWithQueryId(queryId, resultIndex, null); } /** Simulate EMR-S bulk writes query_execution_result with refresh = wait_for */ diff --git a/async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java b/async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java index 66230464e5..4de3a56dd9 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java @@ -29,6 +29,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; @ExtendWith(MockitoExtension.class) public class OpenSearchJobExecutionResponseReaderTest { @@ -50,7 +51,11 @@ public void testGetResultFromOpensearchIndex() { new SearchHit[] {searchHit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F)); Mockito.when(searchHit.getSourceAsMap()).thenReturn(Map.of("stepId", EMR_JOB_ID)); - assertFalse(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null).isEmpty()); + assertFalse( + jobExecutionResponseReader + .getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).build(), null) + .isEmpty()); } @Test @@ -64,7 +69,11 @@ public void testGetResultFromCustomIndex() { new SearchHit[] {searchHit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F)); Mockito.when(searchHit.getSourceAsMap()).thenReturn(Map.of("stepId", EMR_JOB_ID)); - assertFalse(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, "foo").isEmpty()); + assertFalse( + jobExecutionResponseReader + .getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).resultIndex("foo").build(), null) + .isEmpty()); } @Test @@ -76,7 +85,9 @@ public void testInvalidSearchResponse() { RuntimeException exception = assertThrows( RuntimeException.class, - () -> jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)); + () -> + jobExecutionResponseReader.getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).build(), null)); Assertions.assertEquals( "Fetching result from " @@ -92,13 +103,18 @@ public void testSearchFailure() { assertThrows( RuntimeException.class, - () -> jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)); + () -> + jobExecutionResponseReader.getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).build(), null)); } @Test public void testIndexNotFoundException() { when(client.search(any())).thenThrow(IndexNotFoundException.class); - - assertTrue(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, "foo").isEmpty()); + assertTrue( + jobExecutionResponseReader + .getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).resultIndex("foo").build(), null) + .isEmpty()); } } From 69bacd1915d423d364b0085a0782ded229aa29c4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:02:06 -0700 Subject: [PATCH 14/32] [Backport 2.x] Add pit for pagination query (#3076) * Add pit for join queries (#2703) * Add search after for join * Enable search after by default * Add pit * nit * Fix tests * ignore joinWithGeoIntersectNL * Rerun CI with scroll * Remove unused code and retrigger CI with search_after true * Address comments * Remove unused code change * Update pit keep alive time with SQL_CURSOR_KEEP_ALIVE * Fix scroll condition * nit * Add pit before query execution * nit * Move pit from join request builder to executor * Remove unused methods * Add pit in parent class's run() * Add comment for fetching subsequent result in NestedLoopsElasticExecutor * Update comment * Add javadoc for pit handler * Add pit interface * Add pit handler unit test * Fix failed unit test CI * Fix spotless error * Rename pit class and add logs * Fix pit delete unit test --------- * Add pit for multi query (#2753) * Add search after for join * Enable search after by default * Add pit * nit * Fix tests * ignore joinWithGeoIntersectNL * Rerun CI with scroll * draft * Remove unused code and retrigger CI with search_after true * Address comments * Remove unused code change * Update pit keep alive time with SQL_CURSOR_KEEP_ALIVE * Fix scroll condition * nit * Add pit before query execution * Refactor get response with pit method * Update remaining scroll search calls * Fix integ test failures * nit * Move pit from join request builder to executor * Remove unused methods * Move pit from request to executor * Fix pit.delete call missed while merge * Move getResponseWithHits method to util class * add try catch for create delete pit in minus executor * move all common fields to ElasticHitsExecutor * add javadoc for ElasticHitsExecutor * Add missing javadoc * Forcing an empty commit as last commit is stuck processing updates --------- * Add pit to default cursor * Run CI without pit unit test * Rerun CI without pit unit test * FIx unit tests for PIT changes * Addressed comments --------- (cherry picked from commit 69853fe99c4ee940fc739373e0ba04418bda71c6) Signed-off-by: Rupal Mahajan Signed-off-by: Manasvini B S Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] Co-authored-by: Rupal Mahajan --- .../sql/legacy/cursor/DefaultCursor.java | 122 +++++++++++++++++- .../executor/cursor/CursorCloseExecutor.java | 30 ++++- .../executor/cursor/CursorResultExecutor.java | 62 +++++++-- .../format/PrettyFormatRestExecutor.java | 45 ++++++- .../executor/format/SelectResultSet.java | 28 +++- .../executor/join/ElasticJoinExecutor.java | 9 +- .../legacy/executor/multi/MinusExecutor.java | 9 +- .../legacy/pit/PointInTimeHandlerImpl.java | 63 ++++----- .../sql/legacy/query/DefaultQueryAction.java | 16 ++- .../pit/PointInTimeHandlerImplTest.java | 118 ++++++++++++----- .../unittest/cursor/DefaultCursorTest.java | 58 ++++++++- .../query/DefaultQueryActionTest.java | 63 +++++++-- 12 files changed, 513 insertions(+), 110 deletions(-) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/cursor/DefaultCursor.java b/legacy/src/main/java/org/opensearch/sql/legacy/cursor/DefaultCursor.java index 72addd6032..effdb27683 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/cursor/DefaultCursor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/cursor/DefaultCursor.java @@ -5,8 +5,19 @@ package org.opensearch.sql.legacy.cursor; +import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Base64; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -18,6 +29,16 @@ import lombok.Setter; import org.json.JSONArray; import org.json.JSONObject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.search.SearchModule; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.executor.format.Schema; /** @@ -40,6 +61,10 @@ public class DefaultCursor implements Cursor { private static final String SCROLL_ID = "s"; private static final String SCHEMA_COLUMNS = "c"; private static final String FIELD_ALIAS_MAP = "a"; + private static final String PIT_ID = "p"; + private static final String SEARCH_REQUEST = "r"; + private static final String SORT_FIELDS = "h"; + private static final ObjectMapper objectMapper = new ObjectMapper(); /** * To get mappings for index to check if type is date needed for @@ -70,11 +95,28 @@ public class DefaultCursor implements Cursor { /** To get next batch of result */ private String scrollId; + /** To get Point In Time */ + private String pitId; + + /** To get next batch of result with search after api */ + private SearchSourceBuilder searchSourceBuilder; + + /** To get last sort values * */ + private Object[] sortFields; + /** To reduce the number of rows left by fetchSize */ @NonNull private Integer fetchSize; private Integer limit; + /** + * {@link NamedXContentRegistry} from {@link SearchModule} used for construct {@link QueryBuilder} + * from DSL query string. + */ + private static final NamedXContentRegistry xContentRegistry = + new NamedXContentRegistry( + new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()); + @Override public CursorType getType() { return type; @@ -82,19 +124,56 @@ public CursorType getType() { @Override public String generateCursorId() { - if (rowsLeft <= 0 || Strings.isNullOrEmpty(scrollId)) { + if (rowsLeft <= 0 || isCursorIdNullOrEmpty()) { return null; } JSONObject json = new JSONObject(); json.put(FETCH_SIZE, fetchSize); json.put(ROWS_LEFT, rowsLeft); json.put(INDEX_PATTERN, indexPattern); - json.put(SCROLL_ID, scrollId); json.put(SCHEMA_COLUMNS, getSchemaAsJson()); json.put(FIELD_ALIAS_MAP, fieldAliasMap); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + json.put(PIT_ID, pitId); + String sortFieldValue = + AccessController.doPrivileged( + (PrivilegedAction) + () -> { + try { + return objectMapper.writeValueAsString(sortFields); + } catch (JsonProcessingException e) { + throw new RuntimeException( + "Failed to parse sort fields from JSON string.", e); + } + }); + json.put(SORT_FIELDS, sortFieldValue); + setSearchRequestString(json, searchSourceBuilder); + } else { + json.put(SCROLL_ID, scrollId); + } return String.format("%s:%s", type.getId(), encodeCursor(json)); } + private void setSearchRequestString(JSONObject cursorJson, SearchSourceBuilder sourceBuilder) { + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + XContentBuilder builder = XContentFactory.jsonBuilder(outputStream); + sourceBuilder.toXContent(builder, null); + builder.close(); + + String searchRequestBase64 = Base64.getEncoder().encodeToString(outputStream.toByteArray()); + cursorJson.put("searchSourceBuilder", searchRequestBase64); + } catch (IOException ex) { + throw new RuntimeException("Failed to set search request string on cursor json.", ex); + } + } + + private boolean isCursorIdNullOrEmpty() { + return LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER) + ? Strings.isNullOrEmpty(pitId) + : Strings.isNullOrEmpty(scrollId); + } + public static DefaultCursor from(String cursorId) { /** * It is assumed that cursorId here is the second part of the original cursor passed by the @@ -105,13 +184,50 @@ public static DefaultCursor from(String cursorId) { cursor.setFetchSize(json.getInt(FETCH_SIZE)); cursor.setRowsLeft(json.getLong(ROWS_LEFT)); cursor.setIndexPattern(json.getString(INDEX_PATTERN)); - cursor.setScrollId(json.getString(SCROLL_ID)); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + populateCursorForPit(json, cursor); + } else { + cursor.setScrollId(json.getString(SCROLL_ID)); + } cursor.setColumns(getColumnsFromSchema(json.getJSONArray(SCHEMA_COLUMNS))); cursor.setFieldAliasMap(fieldAliasMap(json.getJSONObject(FIELD_ALIAS_MAP))); return cursor; } + private static void populateCursorForPit(JSONObject json, DefaultCursor cursor) { + cursor.setPitId(json.getString(PIT_ID)); + + cursor.setSortFields(getSortFieldsFromJson(json)); + + // Retrieve and set the SearchSourceBuilder from the JSON field + String searchSourceBuilderBase64 = json.getString("searchSourceBuilder"); + byte[] bytes = Base64.getDecoder().decode(searchSourceBuilderBase64); + ByteArrayInputStream streamInput = new ByteArrayInputStream(bytes); + try { + XContentParser parser = + XContentType.JSON + .xContent() + .createParser(xContentRegistry, IGNORE_DEPRECATIONS, streamInput); + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.fromXContent(parser); + cursor.setSearchSourceBuilder(sourceBuilder); + } catch (IOException ex) { + throw new RuntimeException("Failed to get searchSourceBuilder from cursor Id", ex); + } + } + + private static Object[] getSortFieldsFromJson(JSONObject json) { + return AccessController.doPrivileged( + (PrivilegedAction) + () -> { + try { + return objectMapper.readValue(json.getString(SORT_FIELDS), Object[].class); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to parse sort fields from JSON string.", e); + } + }); + } + private JSONArray getSchemaAsJson() { JSONArray schemaJson = new JSONArray(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java index 7282eaed4c..222ca5d9fc 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy.executor.cursor; import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; import java.util.Map; import org.apache.logging.log4j.LogManager; @@ -18,8 +19,11 @@ import org.opensearch.rest.RestChannel; import org.opensearch.sql.legacy.cursor.CursorType; import org.opensearch.sql.legacy.cursor.DefaultCursor; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.rewriter.matchtoterm.VerificationException; public class CursorCloseExecutor implements CursorRestExecutor { @@ -79,14 +83,26 @@ public String execute(Client client, Map params) throws Exceptio } private String handleDefaultCursorCloseRequest(Client client, DefaultCursor cursor) { - String scrollId = cursor.getScrollId(); - ClearScrollResponse clearScrollResponse = - client.prepareClearScroll().addScrollId(scrollId).get(); - if (clearScrollResponse.isSucceeded()) { - return SUCCEEDED_TRUE; + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + String pitId = cursor.getPitId(); + PointInTimeHandler pit = new PointInTimeHandlerImpl(client, pitId); + try { + pit.delete(); + return SUCCEEDED_TRUE; + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + return SUCCEEDED_FALSE; + } } else { - Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - return SUCCEEDED_FALSE; + String scrollId = cursor.getScrollId(); + ClearScrollResponse clearScrollResponse = + client.prepareClearScroll().addScrollId(scrollId).get(); + if (clearScrollResponse.isSucceeded()) { + return SUCCEEDED_TRUE; + } else { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + return SUCCEEDED_FALSE; + } } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java index 854a40b4dd..bc51cff8fd 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java @@ -6,6 +6,8 @@ package org.opensearch.sql.legacy.executor.cursor; import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; import java.util.Arrays; import java.util.Map; @@ -14,6 +16,7 @@ import org.json.JSONException; import org.opensearch.OpenSearchException; import org.opensearch.action.search.ClearScrollResponse; +import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.unit.TimeValue; @@ -21,7 +24,8 @@ import org.opensearch.rest.RestChannel; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; -import org.opensearch.sql.common.setting.Settings; +import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.sql.legacy.cursor.CursorType; import org.opensearch.sql.legacy.cursor.DefaultCursor; import org.opensearch.sql.legacy.esdomain.LocalClusterState; @@ -29,6 +33,8 @@ import org.opensearch.sql.legacy.executor.format.Protocol; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.rewriter.matchtoterm.VerificationException; public class CursorResultExecutor implements CursorRestExecutor { @@ -91,14 +97,27 @@ public String execute(Client client, Map params) throws Exceptio } private String handleDefaultCursorRequest(Client client, DefaultCursor cursor) { - String previousScrollId = cursor.getScrollId(); LocalClusterState clusterState = LocalClusterState.state(); - TimeValue scrollTimeout = clusterState.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE); - SearchResponse scrollResponse = - client.prepareSearchScroll(previousScrollId).setScroll(scrollTimeout).get(); + TimeValue paginationTimeout = clusterState.getSettingValue(SQL_CURSOR_KEEP_ALIVE); + + SearchResponse scrollResponse = null; + if (clusterState.getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + String pitId = cursor.getPitId(); + SearchSourceBuilder source = cursor.getSearchSourceBuilder(); + source.searchAfter(cursor.getSortFields()); + source.pointInTimeBuilder(new PointInTimeBuilder(pitId)); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(source); + scrollResponse = client.search(searchRequest).actionGet(); + } else { + String previousScrollId = cursor.getScrollId(); + scrollResponse = + client.prepareSearchScroll(previousScrollId).setScroll(paginationTimeout).get(); + } SearchHits searchHits = scrollResponse.getHits(); SearchHit[] searchHitArray = searchHits.getHits(); String newScrollId = scrollResponse.getScrollId(); + String newPitId = scrollResponse.pointInTimeId(); int rowsLeft = (int) cursor.getRowsLeft(); int fetch = cursor.getFetchSize(); @@ -120,16 +139,37 @@ private String handleDefaultCursorRequest(Client client, DefaultCursor cursor) { if (rowsLeft <= 0) { /** Clear the scroll context on last page */ - ClearScrollResponse clearScrollResponse = - client.prepareClearScroll().addScrollId(newScrollId).get(); - if (!clearScrollResponse.isSucceeded()) { - Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.info("Error closing the cursor context {} ", newScrollId); + if (newScrollId != null) { + ClearScrollResponse clearScrollResponse = + client.prepareClearScroll().addScrollId(newScrollId).get(); + if (!clearScrollResponse.isSucceeded()) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error closing the cursor context {} ", newScrollId); + } + } + if (newPitId != null) { + PointInTimeHandler pit = new PointInTimeHandlerImpl(client, newPitId); + try { + pit.delete(); + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error deleting point in time {} ", newPitId); + } } } cursor.setRowsLeft(rowsLeft); - cursor.setScrollId(newScrollId); + if (clusterState.getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + cursor.setPitId(newPitId); + cursor.setSearchSourceBuilder(cursor.getSearchSourceBuilder()); + cursor.setSortFields( + scrollResponse + .getHits() + .getAt(scrollResponse.getHits().getHits().length - 1) + .getSortValues()); + } else { + cursor.setScrollId(newScrollId); + } Protocol protocol = new Protocol(client, searchHits, format.name().toLowerCase(), cursor); return protocol.cursorFormat(); } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java index 00feabf5d8..5f758e7d87 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java @@ -5,23 +5,31 @@ package org.opensearch.sql.legacy.executor.format; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.core.common.Strings; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; +import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.sql.legacy.cursor.Cursor; import org.opensearch.sql.legacy.cursor.DefaultCursor; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.QueryActionElasticExecutor; import org.opensearch.sql.legacy.executor.RestExecutor; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.DefaultQueryAction; import org.opensearch.sql.legacy.query.QueryAction; +import org.opensearch.sql.legacy.query.SqlOpenSearchRequestBuilder; import org.opensearch.sql.legacy.query.join.BackOffRetryStrategy; public class PrettyFormatRestExecutor implements RestExecutor { @@ -90,15 +98,32 @@ public String execute(Client client, Map params, QueryAction que private Protocol buildProtocolForDefaultQuery(Client client, DefaultQueryAction queryAction) throws SqlParseException { - SearchResponse response = (SearchResponse) queryAction.explain().get(); - String scrollId = response.getScrollId(); + PointInTimeHandler pit = null; + SearchResponse response; + SqlOpenSearchRequestBuilder sqlOpenSearchRequestBuilder = queryAction.explain(); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit = new PointInTimeHandlerImpl(client, queryAction.getSelect().getIndexArr()); + pit.create(); + SearchRequestBuilder searchRequest = queryAction.getRequestBuilder(); + searchRequest.setPointInTime(new PointInTimeBuilder(pit.getPitId())); + response = searchRequest.get(); + } else { + response = (SearchResponse) sqlOpenSearchRequestBuilder.get(); + } Protocol protocol; - if (!Strings.isNullOrEmpty(scrollId)) { + if (isDefaultCursor(response, queryAction)) { DefaultCursor defaultCursor = new DefaultCursor(); - defaultCursor.setScrollId(scrollId); defaultCursor.setLimit(queryAction.getSelect().getRowCount()); defaultCursor.setFetchSize(queryAction.getSqlRequest().fetchSize()); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + defaultCursor.setPitId(pit.getPitId()); + defaultCursor.setSearchSourceBuilder(queryAction.getRequestBuilder().request().source()); + defaultCursor.setSortFields( + response.getHits().getAt(response.getHits().getHits().length - 1).getSortValues()); + } else { + defaultCursor.setScrollId(response.getScrollId()); + } protocol = new Protocol(client, queryAction, response.getHits(), format, defaultCursor); } else { protocol = new Protocol(client, queryAction, response.getHits(), format, Cursor.NULL_CURSOR); @@ -106,4 +131,16 @@ private Protocol buildProtocolForDefaultQuery(Client client, DefaultQueryAction return protocol; } + + private boolean isDefaultCursor(SearchResponse searchResponse, DefaultQueryAction queryAction) { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + if (searchResponse.getHits().getTotalHits().value < queryAction.getSqlRequest().fetchSize()) { + return false; + } else { + return true; + } + } else { + return !Strings.isNullOrEmpty(searchResponse.getScrollId()); + } + } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index aaf5ef2bc0..1dece1488a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -40,6 +40,7 @@ import org.opensearch.search.aggregations.metrics.NumericMetricsAggregation; import org.opensearch.search.aggregations.metrics.Percentile; import org.opensearch.search.aggregations.metrics.Percentiles; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.cursor.Cursor; import org.opensearch.sql.legacy.cursor.DefaultCursor; import org.opensearch.sql.legacy.domain.ColumnTypeProvider; @@ -49,11 +50,14 @@ import org.opensearch.sql.legacy.domain.Query; import org.opensearch.sql.legacy.domain.Select; import org.opensearch.sql.legacy.domain.TableOnJoinSelect; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.esdomain.mapping.FieldMapping; import org.opensearch.sql.legacy.exception.SqlFeatureNotImplementedException; import org.opensearch.sql.legacy.executor.Format; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.utils.SQLFunctions; public class SelectResultSet extends ResultSet { @@ -564,12 +568,24 @@ private void populateDefaultCursor(DefaultCursor cursor) { long rowsLeft = rowsLeft(cursor.getFetchSize(), cursor.getLimit()); if (rowsLeft <= 0) { // close the cursor - String scrollId = cursor.getScrollId(); - ClearScrollResponse clearScrollResponse = - client.prepareClearScroll().addScrollId(scrollId).get(); - if (!clearScrollResponse.isSucceeded()) { - Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.error("Error closing the cursor context {} ", scrollId); + if (LocalClusterState.state().getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) { + String pitId = cursor.getPitId(); + PointInTimeHandler pit = new PointInTimeHandlerImpl(client, pitId); + try { + pit.delete(); + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error deleting point in time {} ", pitId); + } + } else { + // close the cursor + String scrollId = cursor.getScrollId(); + ClearScrollResponse clearScrollResponse = + client.prepareClearScroll().addScrollId(scrollId).get(); + if (!clearScrollResponse.isSucceeded()) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.error("Error closing the cursor context {} ", scrollId); + } } return; } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java index 061868c9b5..c589edcf50 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -31,6 +31,8 @@ import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.ElasticHitsExecutor; +import org.opensearch.sql.legacy.metrics.MetricName; +import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.SqlElasticRequestBuilder; import org.opensearch.sql.legacy.query.join.HashJoinElasticRequestBuilder; @@ -99,7 +101,12 @@ public void run() throws IOException, SqlParseException { LOG.error("Failed during join query run.", e); } finally { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { - pit.delete(); + try { + pit.delete(); + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error deleting point in time {} ", pit); + } } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java index f58b25e821..06186d0695 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java @@ -33,6 +33,8 @@ import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.ElasticHitsExecutor; +import org.opensearch.sql.legacy.metrics.MetricName; +import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.DefaultQueryAction; import org.opensearch.sql.legacy.query.multi.MultiQueryRequestBuilder; @@ -120,7 +122,12 @@ public void run() throws SqlParseException { LOG.error("Failed during multi query run.", e); } finally { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { - pit.delete(); + try { + pit.delete(); + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error deleting point in time {} ", pit); + } } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java index 64535749e8..8d61c03388 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java @@ -7,16 +7,19 @@ import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; +import java.util.concurrent.ExecutionException; import lombok.Getter; import lombok.Setter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.CreatePitAction; import org.opensearch.action.search.CreatePitRequest; import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.DeletePitAction; import org.opensearch.action.search.DeletePitRequest; import org.opensearch.action.search.DeletePitResponse; import org.opensearch.client.Client; -import org.opensearch.core.action.ActionListener; +import org.opensearch.common.action.ActionFuture; import org.opensearch.sql.legacy.esdomain.LocalClusterState; /** Handler for Point In Time */ @@ -37,47 +40,45 @@ public PointInTimeHandlerImpl(Client client, String[] indices) { this.indices = indices; } + /** + * Constructor for class + * + * @param client OpenSearch client + * @param pitId Point In Time ID + */ + public PointInTimeHandlerImpl(Client client, String pitId) { + this.client = client; + this.pitId = pitId; + } + /** Create PIT for given indices */ @Override public void create() { CreatePitRequest createPitRequest = new CreatePitRequest( LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE), false, indices); - client.createPit( - createPitRequest, - new ActionListener<>() { - @Override - public void onResponse(CreatePitResponse createPitResponse) { - pitId = createPitResponse.getId(); - LOG.info("Created Point In Time {} successfully.", pitId); - } - - @Override - public void onFailure(Exception e) { - LOG.error("Error occurred while creating PIT", e); - } - }); + ActionFuture execute = + client.execute(CreatePitAction.INSTANCE, createPitRequest); + try { + CreatePitResponse pitResponse = execute.get(); + pitId = pitResponse.getId(); + LOG.info("Created Point In Time {} successfully.", pitId); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Error occurred while creating PIT.", e); + } } /** Delete PIT */ @Override public void delete() { DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); - client.deletePits( - deletePitRequest, - new ActionListener<>() { - @Override - public void onResponse(DeletePitResponse deletePitResponse) { - LOG.info( - "Delete Point In Time {} status: {}", - pitId, - deletePitResponse.status().getStatus()); - } - - @Override - public void onFailure(Exception e) { - LOG.error("Error occurred while deleting PIT", e); - } - }); + ActionFuture execute = + client.execute(DeletePitAction.INSTANCE, deletePitRequest); + try { + DeletePitResponse deletePitResponse = execute.get(); + LOG.info("Delete Point In Time {} status: {}", pitId, deletePitResponse.status().getStatus()); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Error occurred while deleting PIT.", e); + } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java index 18c9708df8..9877b17a8f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java @@ -5,6 +5,8 @@ package org.opensearch.sql.legacy.query; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; @@ -100,7 +102,19 @@ public void checkAndSetScroll() { .getNumericalMetric(MetricName.DEFAULT_CURSOR_REQUEST_COUNT_TOTAL) .increment(); Metrics.getInstance().getNumericalMetric(MetricName.DEFAULT_CURSOR_REQUEST_TOTAL).increment(); - request.setSize(fetchSize).setScroll(timeValue); + request.setSize(fetchSize); + // Set scroll or search after for pagination + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + // search after requires results to be in specific order + // set sort field for search_after + boolean ordered = select.isOrderdSelect(); + if (!ordered) { + request.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + } + // Request also requires PointInTime, but we should create pit while execution. + } else { + request.setScroll(timeValue); + } } else { request.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); setLimit(select.getOffset(), rowCount != null ? rowCount : Select.DEFAULT_LIMIT); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java index 42f1af4563..eba4ae0346 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java @@ -2,80 +2,132 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.legacy.pit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; -import java.util.concurrent.CompletableFuture; +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import lombok.SneakyThrows; import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.opensearch.action.search.CreatePitAction; +import org.opensearch.action.search.CreatePitRequest; import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.DeletePitAction; +import org.opensearch.action.search.DeletePitRequest; import org.opensearch.action.search.DeletePitResponse; import org.opensearch.client.Client; +import org.opensearch.common.action.ActionFuture; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.esdomain.LocalClusterState; +import org.opensearch.sql.opensearch.setting.OpenSearchSettings; public class PointInTimeHandlerImplTest { @Mock private Client mockClient; private String[] indices = {"index1", "index2"}; private PointInTimeHandlerImpl pointInTimeHandlerImpl; - @Captor private ArgumentCaptor> listenerCaptor; private final String PIT_ID = "testId"; + private CreatePitResponse mockCreatePitResponse; + private DeletePitResponse mockDeletePitResponse; + private ActionFuture mockActionFuture; + private ActionFuture mockActionFutureDelete; + + @Mock private OpenSearchSettings settings; @Before public void setUp() { MockitoAnnotations.initMocks(this); pointInTimeHandlerImpl = new PointInTimeHandlerImpl(mockClient, indices); - } - @Test - public void testCreate() { - when(LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE)) + doReturn(Collections.emptyList()).when(settings).getSettings(); + when(settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE)) .thenReturn(new TimeValue(10000)); + LocalClusterState.state().setPluginSettings(settings); - CreatePitResponse mockCreatePitResponse = mock(CreatePitResponse.class); + mockCreatePitResponse = mock(CreatePitResponse.class); + mockDeletePitResponse = mock(DeletePitResponse.class); + mockActionFuture = mock(ActionFuture.class); + mockActionFutureDelete = mock(ActionFuture.class); + when(mockClient.execute(any(CreatePitAction.class), any(CreatePitRequest.class))) + .thenReturn(mockActionFuture); + when(mockClient.execute(any(DeletePitAction.class), any(DeletePitRequest.class))) + .thenReturn(mockActionFutureDelete); + RestStatus mockRestStatus = mock(RestStatus.class); + when(mockDeletePitResponse.status()).thenReturn(mockRestStatus); + when(mockDeletePitResponse.status().getStatus()).thenReturn(200); when(mockCreatePitResponse.getId()).thenReturn(PIT_ID); + } - CompletableFuture completableFuture = - CompletableFuture.completedFuture(mockCreatePitResponse); + @SneakyThrows + @Test + public void testCreate() { + when(mockActionFuture.get()).thenReturn(mockCreatePitResponse); + try { + pointInTimeHandlerImpl.create(); + } catch (RuntimeException e) { + fail("Expected no exception while creating PIT, but got: " + e.getMessage()); + } + verify(mockClient).execute(any(CreatePitAction.class), any(CreatePitRequest.class)); + verify(mockActionFuture).get(); + verify(mockCreatePitResponse).getId(); + } - doAnswer( - invocation -> { - ActionListener actionListener = invocation.getArgument(1); - actionListener.onResponse(mockCreatePitResponse); - return completableFuture; - }) - .when(mockClient) - .createPit(any(), any()); + @SneakyThrows + @Test + public void testCreateForFailure() { + ExecutionException executionException = + new ExecutionException("Error occurred while creating PIT.", new Throwable()); + when(mockActionFuture.get()).thenThrow(executionException); - pointInTimeHandlerImpl.create(); + RuntimeException thrownException = + assertThrows(RuntimeException.class, () -> pointInTimeHandlerImpl.create()); - assertEquals(PIT_ID, pointInTimeHandlerImpl.getPitId()); + verify(mockClient).execute(any(CreatePitAction.class), any(CreatePitRequest.class)); + assertNotNull(thrownException.getCause()); + assertEquals("Error occurred while creating PIT.", thrownException.getMessage()); + verify(mockActionFuture).get(); } + @SneakyThrows @Test public void testDelete() { - DeletePitResponse mockedResponse = mock(DeletePitResponse.class); - RestStatus mockRestStatus = mock(RestStatus.class); - when(mockedResponse.status()).thenReturn(mockRestStatus); - when(mockedResponse.status().getStatus()).thenReturn(200); - pointInTimeHandlerImpl.setPitId(PIT_ID); - pointInTimeHandlerImpl.delete(); - verify(mockClient).deletePits(any(), listenerCaptor.capture()); - listenerCaptor.getValue().onResponse(mockedResponse); + when(mockActionFutureDelete.get()).thenReturn(mockDeletePitResponse); + try { + pointInTimeHandlerImpl.delete(); + } catch (RuntimeException e) { + fail("Expected no exception while deleting PIT, but got: " + e.getMessage()); + } + verify(mockClient).execute(any(DeletePitAction.class), any(DeletePitRequest.class)); + verify(mockActionFutureDelete).get(); + } + + @SneakyThrows + @Test + public void testDeleteForFailure() { + ExecutionException executionException = + new ExecutionException("Error occurred while deleting PIT.", new Throwable()); + when(mockActionFutureDelete.get()).thenThrow(executionException); + + RuntimeException thrownException = + assertThrows(RuntimeException.class, () -> pointInTimeHandlerImpl.delete()); + + verify(mockClient).execute(any(DeletePitAction.class), any(DeletePitRequest.class)); + assertNotNull(thrownException.getCause()); + assertEquals("Error occurred while deleting PIT.", thrownException.getMessage()); + verify(mockActionFutureDelete).get(); } } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/cursor/DefaultCursorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/cursor/DefaultCursorTest.java index 1b9662035d..deff7132b0 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/cursor/DefaultCursorTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/cursor/DefaultCursorTest.java @@ -9,14 +9,47 @@ import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.Matchers.startsWith; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.Collections; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.cursor.CursorType; import org.opensearch.sql.legacy.cursor.DefaultCursor; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; +import org.opensearch.sql.opensearch.setting.OpenSearchSettings; public class DefaultCursorTest { + @Mock private OpenSearchSettings settings; + + @Mock private SearchSourceBuilder sourceBuilder; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + // Required for Pagination queries using PIT instead of Scroll + doReturn(Collections.emptyList()).when(settings).getSettings(); + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + LocalClusterState.state().setPluginSettings(settings); + + // Mock the toXContent method of SearchSourceBuilder + try { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(new ByteArrayOutputStream()); + when(sourceBuilder.toXContent(any(XContentBuilder.class), any())).thenReturn(xContentBuilder); + } catch (Exception e) { + throw new RuntimeException(e); + } + } @Test public void checkCursorType() { @@ -25,7 +58,26 @@ public void checkCursorType() { } @Test - public void cursorShouldStartWithCursorTypeID() { + public void cursorShouldStartWithCursorTypeIDForPIT() { + DefaultCursor cursor = new DefaultCursor(); + cursor.setRowsLeft(50); + cursor.setPitId("dbdskbcdjksbcjkdsbcjk+//"); + cursor.setIndexPattern("myIndex"); + cursor.setFetchSize(500); + cursor.setFieldAliasMap(Collections.emptyMap()); + cursor.setColumns(new ArrayList<>()); + + // Set the mocked SearchSourceBuilder to the cursor + cursor.setSearchSourceBuilder(sourceBuilder); + + assertThat(cursor.generateCursorId(), startsWith(cursor.getType().getId() + ":")); + } + + @Test + public void cursorShouldStartWithCursorTypeIDForScroll() { + // Disable PIT for pagination and use scroll instead + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + DefaultCursor cursor = new DefaultCursor(); cursor.setRowsLeft(50); cursor.setScrollId("dbdskbcdjksbcjkdsbcjk+//"); @@ -33,6 +85,10 @@ public void cursorShouldStartWithCursorTypeID() { cursor.setFetchSize(500); cursor.setFieldAliasMap(Collections.emptyMap()); cursor.setColumns(new ArrayList<>()); + + // Set the mocked SearchSourceBuilder to the cursor + cursor.setSearchSourceBuilder(sourceBuilder); + assertThat(cursor.generateCursorId(), startsWith(cursor.getType().getId() + ":")); } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java index 755d604a65..ec6ab00f97 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java @@ -8,16 +8,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; +import static org.mockito.Mockito.*; + +import java.util.*; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -27,6 +20,8 @@ import org.opensearch.client.Client; import org.opensearch.common.unit.TimeValue; import org.opensearch.script.Script; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.domain.Field; import org.opensearch.sql.legacy.domain.KVValue; @@ -149,6 +144,12 @@ public void testIfScrollShouldBeOpenWithDifferentFormats() { queryAction.setFormat(Format.JDBC); queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(settingFetchSize); + Mockito.verify(mockRequestBuilder).addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); + + // Verify setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + mockLocalClusterStateAndIntializeMetricsForScroll(timeValue); + queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setScroll(timeValue); } @@ -168,6 +169,12 @@ public void testIfScrollShouldBeOpen() { mockLocalClusterStateAndInitializeMetrics(timeValue); queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(settingFetchSize); + Mockito.verify(mockRequestBuilder).addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); + + // Verify setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + mockLocalClusterStateAndIntializeMetricsForScroll(timeValue); + queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setScroll(timeValue); } @@ -195,6 +202,12 @@ public void testIfScrollShouldBeOpenWithDifferentFetchSize() { doReturn(mockRequestBuilder).when(mockRequestBuilder).setSize(userFetchSize); queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(20); + Mockito.verify(mockRequestBuilder).addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); + + // Verify setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + mockLocalClusterStateAndIntializeMetricsForScroll(timeValue); + queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setScroll(timeValue); } @@ -216,7 +229,9 @@ public void testIfScrollShouldBeOpenWithDifferentValidFetchSizeAndLimit() { queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(userFetchSize); - Mockito.verify(mockRequestBuilder).setScroll(timeValue); + Mockito.verify(mockRequestBuilder).addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + // Skip setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); /** fetchSize > LIMIT - no scroll */ userFetchSize = 5000; @@ -226,6 +241,14 @@ public void testIfScrollShouldBeOpenWithDifferentValidFetchSizeAndLimit() { queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(limit); Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); + + // Verify setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + mockLocalClusterStateAndIntializeMetricsForScroll(timeValue); + /** fetchSize <= LIMIT - open scroll */ + userFetchSize = 1500; + doReturn(userFetchSize).when(mockSqlRequest).fetchSize(); + queryAction.checkAndSetScroll(); + Mockito.verify(mockRequestBuilder).setScroll(timeValue); } private void mockLocalClusterStateAndInitializeMetrics(TimeValue time) { @@ -236,6 +259,24 @@ private void mockLocalClusterStateAndInitializeMetrics(TimeValue time) { .when(mockLocalClusterState) .getSettingValue(Settings.Key.METRICS_ROLLING_WINDOW); doReturn(2L).when(mockLocalClusterState).getSettingValue(Settings.Key.METRICS_ROLLING_INTERVAL); + doReturn(true) + .when(mockLocalClusterState) + .getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER); + + Metrics.getInstance().registerDefaultMetrics(); + } + + private void mockLocalClusterStateAndIntializeMetricsForScroll(TimeValue time) { + LocalClusterState mockLocalClusterState = mock(LocalClusterState.class); + LocalClusterState.state(mockLocalClusterState); + doReturn(time).when(mockLocalClusterState).getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE); + doReturn(3600L) + .when(mockLocalClusterState) + .getSettingValue(Settings.Key.METRICS_ROLLING_WINDOW); + doReturn(2L).when(mockLocalClusterState).getSettingValue(Settings.Key.METRICS_ROLLING_INTERVAL); + doReturn(false) + .when(mockLocalClusterState) + .getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER); Metrics.getInstance().registerDefaultMetrics(); } From 24b90abcc2579498d7ba3b1fe5acdf0428feb9b9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:31:54 -0400 Subject: [PATCH 15/32] Fix grammar in log warnings. (#2928) (#3088) (cherry picked from commit 0e3e57c2d6a83f63e00c6cb6abf94ce1d6963bed) Signed-off-by: dblock Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java | 2 +- .../opensearch/executor/protector/ResourceMonitorPlan.java | 4 ++-- .../sql/opensearch/monitor/OpenSearchMemoryHealthy.java | 2 +- .../sql/opensearch/executor/ResourceMonitorPlanTest.java | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java index eed2369590..2799ab1016 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java @@ -34,7 +34,7 @@ public void queryExceedResourceLimitShouldFail() throws IOException { assertEquals(500, exception.getResponse().getStatusLine().getStatusCode()); assertThat( exception.getMessage(), - Matchers.containsString("resource is not enough to run the" + " query, quit.")); + Matchers.containsString("insufficient resources to run the query, quit.")); // update plugins.ppl.query.memory_limit to default value 85% updateClusterSettings( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/ResourceMonitorPlan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/ResourceMonitorPlan.java index e3bc48ba72..150a749358 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/ResourceMonitorPlan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/ResourceMonitorPlan.java @@ -44,7 +44,7 @@ public R accept(PhysicalPlanNodeVisitor visitor, C context) { @Override public void open() { if (!this.monitor.isHealthy()) { - throw new IllegalStateException("resource is not enough to run the query, quit."); + throw new IllegalStateException("insufficient resources to run the query, quit."); } delegate.open(); } @@ -68,7 +68,7 @@ public boolean hasNext() { public ExprValue next() { boolean shouldCheck = (++nextCallCount % NUMBER_OF_NEXT_CALL_TO_CHECK == 0); if (shouldCheck && !this.monitor.isHealthy()) { - throw new IllegalStateException("resource is not enough to load next row, quit."); + throw new IllegalStateException("insufficient resources to load next row, quit."); } return delegate.next(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/monitor/OpenSearchMemoryHealthy.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/monitor/OpenSearchMemoryHealthy.java index 4b7b6c5dcb..bc038cb42f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/monitor/OpenSearchMemoryHealthy.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/monitor/OpenSearchMemoryHealthy.java @@ -36,7 +36,7 @@ public boolean isMemoryHealthy(long limitBytes) { } else { log.warn("Memory usage:{} exceed limit:{}", memoryUsage, limitBytes); if (randomFail.shouldFail()) { - log.warn("Fast failure the current request"); + log.warn("Fast failing the current request"); throw new MemoryUsageExceedFastFailureException(); } else { throw new MemoryUsageExceedException(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ResourceMonitorPlanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ResourceMonitorPlanTest.java index 26bcdf6d89..82062bee76 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ResourceMonitorPlanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ResourceMonitorPlanTest.java @@ -47,7 +47,7 @@ void openExceedResourceLimit() { IllegalStateException exception = assertThrows(IllegalStateException.class, () -> monitorPlan.open()); - assertEquals("resource is not enough to run the query, quit.", exception.getMessage()); + assertEquals("insufficient resources to run the query, quit.", exception.getMessage()); } @Test @@ -79,7 +79,7 @@ void nextExceedResourceLimit() { IllegalStateException exception = assertThrows(IllegalStateException.class, () -> monitorPlan.next()); - assertEquals("resource is not enough to load next row, quit.", exception.getMessage()); + assertEquals("insufficient resources to load next row, quit.", exception.getMessage()); } @Test From 515c432cf3bfc8e88207eb30a254ae99b95aeeab Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:38:13 -0700 Subject: [PATCH 16/32] Optimize scheduler interval conf (#3049) (#3081) * Optimize scheduler interval conf * Update OpenSearchAsyncQuerySchedulerConfigComposer.java * Update OpenSearchAsyncQuerySchedulerConfigComposer.java --------- (cherry picked from commit a87061d9a21a59e71365aae18523e47701b24b5e) Signed-off-by: Louis Chu Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../OpenSearchAsyncQuerySchedulerConfigComposer.java | 9 +++++++-- .../OpenSearchAsyncQuerySchedulerConfigComposerTest.java | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java index f791b050a1..28fd4b1b58 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL; import lombok.RequiredArgsConstructor; +import org.opensearch.core.common.Strings; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; @@ -30,7 +31,11 @@ public void compose( settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL); sparkSubmitParameters.setConfigItem( FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED, String.valueOf(externalSchedulerEnabled)); - sparkSubmitParameters.setConfigItem( - FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL, externalSchedulerInterval); + if (!Strings.isNullOrEmpty(externalSchedulerInterval)) { + externalSchedulerInterval = + "\"" + externalSchedulerInterval + "\""; // Wrap the value with double quotes + sparkSubmitParameters.setConfigItem( + FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL, externalSchedulerInterval); + } } } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java index 1556d4db3f..19ab091e25 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java @@ -1,5 +1,6 @@ package org.opensearch.sql.spark.config; +import static org.junit.Assert.assertNull; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -40,7 +41,7 @@ public void testCompose() { verify(sparkSubmitParameters) .setConfigItem("spark.flint.job.externalScheduler.enabled", "true"); verify(sparkSubmitParameters) - .setConfigItem("spark.flint.job.externalScheduler.interval", "10 minutes"); + .setConfigItem("spark.flint.job.externalScheduler.interval", "\"10 minutes\""); } @Test @@ -63,6 +64,6 @@ public void testComposeWithMissingInterval() { composer.compose(sparkSubmitParameters, dispatchQueryRequest, context); - verify(sparkSubmitParameters).setConfigItem("spark.flint.job.externalScheduler.interval", ""); + assertNull(sparkSubmitParameters.getConfigItem("spark.flint.job.externalScheduler.interval")); } } From 25511db55f64b55c45b4b04b96940a02076dbf61 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:08:34 -0400 Subject: [PATCH 17/32] bump commons-io to 2.14.0 (#3083) (#3091) (cherry picked from commit 07101343cf91249c1f2a01f6fc9f96f0ff777ebe) Signed-off-by: Sean Kao Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- async-query/build.gradle | 2 +- datasources/build.gradle | 2 +- spark/build.gradle | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/async-query/build.gradle b/async-query/build.gradle index 53fdcbe292..fba74aa216 100644 --- a/async-query/build.gradle +++ b/async-query/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation group: 'org.json', name: 'json', version: '20231013' api group: 'com.amazonaws', name: 'aws-java-sdk-emr', version: "${aws_java_sdk_version}" api group: 'com.amazonaws', name: 'aws-java-sdk-emrserverless', version: "${aws_java_sdk_version}" - implementation group: 'commons-io', name: 'commons-io', version: '2.8.0' + implementation group: 'commons-io', name: 'commons-io', version: '2.14.0' testImplementation(platform("org.junit:junit-bom:5.9.3")) diff --git a/datasources/build.gradle b/datasources/build.gradle index 9456141e80..1d1127ad0d 100644 --- a/datasources/build.gradle +++ b/datasources/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" implementation group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}" - implementation group: 'commons-io', name: 'commons-io', version: '2.8.0' + implementation group: 'commons-io', name: 'commons-io', version: '2.14.0' // FIXME. upgrade aws-encryption-sdk-java once the bouncycastle dependency update to 1.78. implementation ('com.amazonaws:aws-encryption-sdk-java:2.4.1') { exclude group: 'org.bouncycastle', module: 'bcprov-ext-jdk18on' diff --git a/spark/build.gradle b/spark/build.gradle index d9d5c96413..103c017791 100644 --- a/spark/build.gradle +++ b/spark/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation group: 'org.json', name: 'json', version: '20231013' api group: 'com.amazonaws', name: 'aws-java-sdk-emr', version: "${aws_java_sdk_version}" api group: 'com.amazonaws', name: 'aws-java-sdk-emrserverless', version: "${aws_java_sdk_version}" - implementation group: 'commons-io', name: 'commons-io', version: '2.8.0' + implementation group: 'commons-io', name: 'commons-io', version: '2.14.0' testImplementation(platform("org.junit:junit-bom:5.9.3")) From 6274b88219f34ab06fda4fc34f87bc25d182a378 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:31:36 -0700 Subject: [PATCH 18/32] Remove scheduler index from SystemIndexDescriptor (#3092) (#3097) (cherry picked from commit 366d165680a0674ee630cc7bad1522166641331b) Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 560c5edadd..766edc42c0 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -342,9 +342,6 @@ public Collection getSystemIndexDescriptors(Settings sett systemIndexDescriptors.add( new SystemIndexDescriptor( SPARK_REQUEST_BUFFER_INDEX_NAME + "*", "SQL Spark Request Buffer index pattern")); - systemIndexDescriptors.add( - new SystemIndexDescriptor( - OpenSearchAsyncQueryScheduler.SCHEDULER_INDEX_NAME, "SQL Scheduler job index")); return systemIndexDescriptors; } } From 3cd77021e4d745f8c01af88ddb5244926f659495 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:21:27 -0700 Subject: [PATCH 19/32] [Backport 2.x] Improve error handling for malformed query cursors (#3084) * Add reproducer for malformed cursor handling * Handle malformed query cursors * Move malformed cursor test to correct file * Apply spotless --------- (cherry picked from commit 7477fd8e0a1cf46e31ac2b90e094720fe3d92598) Signed-off-by: Simeon Widdis Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../java/org/opensearch/sql/legacy/CursorIT.java | 11 +++++++++++ .../legacy/executor/cursor/CursorResultExecutor.java | 12 +++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index 1fc532c1f8..4ee99198df 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -423,6 +423,17 @@ public void noPaginationWithNonJDBCFormat() throws IOException { assertThat(rows.length, equalTo(1000)); } + @Test + public void testMalformedCursorGracefullyHandled() throws IOException { + ResponseException result = + assertThrows( + "Expected query with malformed cursor to raise error, but didn't", + ResponseException.class, + () -> executeCursorQuery("d:a11b4db33f")); + assertTrue(result.getMessage().contains("Malformed cursor")); + assertEquals(result.getResponse().getStatusLine().getStatusCode(), 400); + } + public void verifyWithAndWithoutPaginationResponse( String sqlQuery, String cursorQuery, int fetch_size, boolean shouldFallBackToV1) throws IOException { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java index bc51cff8fd..6903652363 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java @@ -20,6 +20,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.search.SearchHit; @@ -36,6 +37,7 @@ import org.opensearch.sql.legacy.pit.PointInTimeHandler; import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.rewriter.matchtoterm.VerificationException; +import org.opensearch.sql.opensearch.response.error.ErrorMessageFactory; public class CursorResultExecutor implements CursorRestExecutor { @@ -58,7 +60,15 @@ public void execute(Client client, Map params, RestChannel chann } catch (IllegalArgumentException | JSONException e) { Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_CUS).increment(); LOG.error("Error parsing the cursor", e); - channel.sendResponse(new BytesRestResponse(channel, e)); + channel.sendResponse( + new BytesRestResponse( + RestStatus.BAD_REQUEST, + "application/json; charset=UTF-8", + ErrorMessageFactory.createErrorMessage( + new IllegalArgumentException( + "Malformed cursor: unable to extract cursor information"), + RestStatus.BAD_REQUEST.getStatus()) + .toString())); } catch (OpenSearchException e) { int status = (e.status().getStatus()); if (status > 399 && status < 500) { From 79ebf6319d01b3655eefd726324ecefb2b9929e7 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:41:47 -0700 Subject: [PATCH 20/32] [Backport 2.x] Correct regular expression range (#2910) (cherry picked from commit 7022a0946015c41cbd02888e3c712c6b67123982) Signed-off-by: Lantao Jin Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../java/org/opensearch/sql/common/grok/GrokCompiler.java | 2 +- .../main/java/org/opensearch/sql/common/grok/GrokUtils.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java b/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java index aba96ad4cb..05fdbd57ed 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java @@ -29,7 +29,7 @@ public class GrokCompiler implements Serializable { // We don't want \n and commented line - private static final Pattern patternLinePattern = Pattern.compile("^([A-z0-9_]+)\\s+(.*)$"); + private static final Pattern patternLinePattern = Pattern.compile("^([a-zA-Z0-9_]+)\\s+(.*)$"); /** {@code Grok} patterns definitions. */ private final Map grokPatternDefinitions = new HashMap<>(); diff --git a/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java b/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java index 4b145bbbe8..2a309bba8f 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java @@ -24,8 +24,8 @@ public class GrokUtils { Pattern.compile( "%\\{" + "(?" - + "(?[A-z0-9]+)" - + "(?::(?[A-z0-9_:;,\\-\\/\\s\\.']+))?" + + "(?[a-zA-Z0-9_]+)" + + "(?::(?[a-zA-Z0-9_:;,\\-\\/\\s\\.']+))?" + ")" + "(?:=(?" + "(?:" From 2d3d3d8a5aa44d4384c50f6f776238c0af77a52d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:42:01 -0700 Subject: [PATCH 21/32] [Backport 2.x] SQL pagination should work with the `pretty` parameter (#2828) (cherry picked from commit 0c2e1da96a5b34337e7ffa971842fcce4a6e10f2) Signed-off-by: Lantao Jin Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] Co-authored-by: Lantao Jin --- .../org/opensearch/sql/jdbc/CursorIT.java | 20 +++++++++++ .../sql/sql/domain/SQLQueryRequest.java | 29 ++++++++------- .../sql/sql/domain/SQLQueryRequestTest.java | 36 +++++++++++++++++++ 3 files changed, 72 insertions(+), 13 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/jdbc/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/jdbc/CursorIT.java index b3f2c1f0ca..56066a4b2b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/jdbc/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/jdbc/CursorIT.java @@ -22,6 +22,7 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.List; +import java.util.Map; import javax.annotation.Nullable; import lombok.SneakyThrows; import org.json.JSONObject; @@ -116,6 +117,8 @@ public void select_all_no_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -134,6 +137,8 @@ public void select_count_all_no_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -152,6 +157,8 @@ public void select_all_small_table_big_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -170,6 +177,8 @@ public void select_all_small_table_small_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -188,6 +197,8 @@ public void select_all_big_table_small_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -206,6 +217,8 @@ public void select_all_big_table_big_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -218,6 +231,12 @@ private static String getConnectionString() { @SneakyThrows protected JSONObject executeRestQuery(String query, @Nullable Integer fetch_size) { + return executeRestQuery(query, fetch_size, Map.of()); + } + + @SneakyThrows + protected JSONObject executeRestQuery( + String query, @Nullable Integer fetch_size, Map params) { Request request = new Request("POST", QUERY_API_ENDPOINT); if (fetch_size != null) { request.setJsonEntity( @@ -225,6 +244,7 @@ protected JSONObject executeRestQuery(String query, @Nullable Integer fetch_size } else { request.setJsonEntity(String.format("{ \"query\": \"%s\" }", query)); } + request.addParameters(params); RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); restOptionsBuilder.addHeader("Content-Type", "application/json"); diff --git a/sql/src/main/java/org/opensearch/sql/sql/domain/SQLQueryRequest.java b/sql/src/main/java/org/opensearch/sql/sql/domain/SQLQueryRequest.java index 8b7c450a87..d87846b382 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/domain/SQLQueryRequest.java +++ b/sql/src/main/java/org/opensearch/sql/sql/domain/SQLQueryRequest.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Stream; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -81,19 +82,21 @@ public SQLQueryRequest( * @return true if supported. */ public boolean isSupported() { - var noCursor = !isCursor(); - var noQuery = query == null; - var noUnsupportedParams = - params.isEmpty() || (params.size() == 1 && params.containsKey(QUERY_PARAMS_FORMAT)); - var noContent = jsonContent == null || jsonContent.isEmpty(); - - return ((!noCursor - && noQuery - && noUnsupportedParams - && noContent) // if cursor is given, but other things - || (noCursor && !noQuery)) // or if cursor is not given, but query - && isOnlySupportedFieldInPayload() // and request has supported fields only - && isSupportedFormat(); // and request is in supported format + boolean hasCursor = isCursor(); + boolean hasQuery = query != null; + boolean hasContent = jsonContent != null && !jsonContent.isEmpty(); + + Predicate supportedParams = Set.of(QUERY_PARAMS_FORMAT, QUERY_PARAMS_PRETTY)::contains; + boolean hasUnsupportedParams = + (!params.isEmpty()) + && params.keySet().stream().dropWhile(supportedParams).findAny().isPresent(); + + boolean validCursor = hasCursor && !hasQuery && !hasUnsupportedParams && !hasContent; + boolean validQuery = !hasCursor && hasQuery; + + return (validCursor || validQuery) // It's a valid cursor or a valid query + && isOnlySupportedFieldInPayload() // and request must contain supported fields only + && isSupportedFormat(); // and request must be a supported format } private boolean isCursor() { diff --git a/sql/src/test/java/org/opensearch/sql/sql/domain/SQLQueryRequestTest.java b/sql/src/test/java/org/opensearch/sql/sql/domain/SQLQueryRequestTest.java index 2b64b13b35..b569a89a2e 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/domain/SQLQueryRequestTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/domain/SQLQueryRequestTest.java @@ -106,6 +106,42 @@ public void should_support_cursor_request() { () -> assertTrue(cursorRequest.isSupported())); } + @Test + public void should_support_cursor_request_with_supported_parameters() { + SQLQueryRequest fetchSizeRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"query\": \"SELECT 1\", \"fetch_size\": 5}") + .build(); + + SQLQueryRequest cursorRequest = + SQLQueryRequestBuilder.request(null) + .cursor("abcdefgh...") + .params(Map.of("format", "csv", "pretty", "true")) + .build(); + + assertAll( + () -> assertTrue(fetchSizeRequest.isSupported()), + () -> assertTrue(cursorRequest.isSupported())); + } + + @Test + public void should_not_support_cursor_request_with_unsupported_parameters() { + SQLQueryRequest fetchSizeRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"query\": \"SELECT 1\", \"fetch_size\": 5}") + .build(); + + SQLQueryRequest cursorRequest = + SQLQueryRequestBuilder.request(null) + .cursor("abcdefgh...") + .params(Map.of("one", "two")) + .build(); + + assertAll( + () -> assertTrue(fetchSizeRequest.isSupported()), + () -> assertFalse(cursorRequest.isSupported())); + } + @Test public void should_support_cursor_close_request() { SQLQueryRequest closeRequest = From e2ea012042c340a93dd7b3d656fc5252ea6aaf16 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:58:48 -0700 Subject: [PATCH 22/32] [Backport 2.x] Bug Fixes for minor issues with SQL PIT refactor (#3100) (cherry picked from commit ec5c3b754e96eb0c3e223880994fd9b39e22d5a6) Signed-off-by: Manasvini B S Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../format/PrettyFormatRestExecutor.java | 11 ++- .../executor/join/ElasticJoinExecutor.java | 1 + .../format/PrettyFormatRestExecutorTest.java | 89 +++++++++++++++++++ 3 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java index 5f758e7d87..3344829859 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; import java.util.Map; +import java.util.Objects; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; @@ -132,13 +133,11 @@ private Protocol buildProtocolForDefaultQuery(Client client, DefaultQueryAction return protocol; } - private boolean isDefaultCursor(SearchResponse searchResponse, DefaultQueryAction queryAction) { + protected boolean isDefaultCursor(SearchResponse searchResponse, DefaultQueryAction queryAction) { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { - if (searchResponse.getHits().getTotalHits().value < queryAction.getSqlRequest().fetchSize()) { - return false; - } else { - return true; - } + return queryAction.getSqlRequest().fetchSize() != 0 + && Objects.requireNonNull(searchResponse.getHits().getTotalHits()).value + >= queryAction.getSqlRequest().fetchSize(); } else { return !Strings.isNullOrEmpty(searchResponse.getScrollId()); } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java index c589edcf50..e5011d1af8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -99,6 +99,7 @@ public void run() throws IOException, SqlParseException { this.metaResults.setTookImMilli(joinTimeInMilli); } catch (Exception e) { LOG.error("Failed during join query run.", e); + throw new IllegalStateException("Error occurred during join query run", e); } finally { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { try { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java new file mode 100644 index 0000000000..1387412d37 --- /dev/null +++ b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java @@ -0,0 +1,89 @@ +package org.opensearch.sql.legacy.executor.format; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + +import org.apache.lucene.search.TotalHits; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; +import org.opensearch.sql.legacy.query.DefaultQueryAction; +import org.opensearch.sql.legacy.request.SqlRequest; +import org.opensearch.sql.opensearch.setting.OpenSearchSettings; + +@RunWith(MockitoJUnitRunner.class) +public class PrettyFormatRestExecutorTest { + + @Mock private SearchResponse searchResponse; + @Mock private SearchHits searchHits; + @Mock private SearchHit searchHit; + @Mock private DefaultQueryAction queryAction; + @Mock private SqlRequest sqlRequest; + private PrettyFormatRestExecutor executor; + + @Before + public void setUp() { + OpenSearchSettings settings = mock(OpenSearchSettings.class); + LocalClusterState.state().setPluginSettings(settings); + when(LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); + when(queryAction.getSqlRequest()).thenReturn(sqlRequest); + executor = new PrettyFormatRestExecutor("jdbc"); + } + + @Test + public void testIsDefaultCursor_fetchSizeZero() { + when(sqlRequest.fetchSize()).thenReturn(0); + + assertFalse(executor.isDefaultCursor(searchResponse, queryAction)); + } + + @Test + public void testIsDefaultCursor_totalHitsLessThanFetchSize() { + when(sqlRequest.fetchSize()).thenReturn(10); + when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, new TotalHits(5, TotalHits.Relation.EQUAL_TO), 1.0F)); + + assertFalse(executor.isDefaultCursor(searchResponse, queryAction)); + } + + @Test + public void testIsDefaultCursor_totalHitsGreaterThanOrEqualToFetchSize() { + when(sqlRequest.fetchSize()).thenReturn(5); + when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, new TotalHits(5, TotalHits.Relation.EQUAL_TO), 1.0F)); + + assertTrue(executor.isDefaultCursor(searchResponse, queryAction)); + } + + @Test + public void testIsDefaultCursor_PaginationApiDisabled() { + when(LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(false); + when(searchResponse.getScrollId()).thenReturn("someScrollId"); + + assertTrue(executor.isDefaultCursor(searchResponse, queryAction)); + } + + @Test + public void testIsDefaultCursor_PaginationApiDisabled_NoScrollId() { + when(LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(false); + when(searchResponse.getScrollId()).thenReturn(null); + + assertFalse(executor.isDefaultCursor(searchResponse, queryAction)); + } +} From 2921f2e30010ec82b75b93595664f0a5dd41d044 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 12:00:57 -0700 Subject: [PATCH 23/32] [Backport 2.x] Resolve Alias Issues in Legacy SQL with Filters (#3059) * Fix: Pagination of index aliases is not supported * fix: remove extra debug log * Integration testadded * rollback change Signed-off-by: Aparajita Pandey * Integration TestAdded Signed-off-by: Aparajita Pandey * Integration TestAdded Signed-off-by: Aparajita Pandey * SpotlessCheck Signed-off-by: Aparajita Pandey --------- (cherry picked from commit 564ab60ba1225dcdf30b129fbef5685979a3ab8a) Signed-off-by: Aparajita Pandey Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../sql/legacy/SQLIntegTestCase.java | 6 +++ .../org/opensearch/sql/sql/PaginationIT.java | 50 +++++++++++++++++++ .../executor/format/SelectResultSet.java | 8 ++- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index f30864d4b1..dd98596ad8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -468,6 +468,12 @@ protected String makeRequest(String query, int fetch_size) { "{\n" + " \"fetch_size\": \"%s\",\n" + " \"query\": \"%s\"\n" + "}", fetch_size, query); } + protected String makeRequest(String query, int fetch_size, String filterQuery) { + return String.format( + "{ \"fetch_size\": \"%s\", \"query\": \"%s\", \"filter\" : %s }", + fetch_size, query, filterQuery); + } + protected String makeFetchLessRequest(String query) { return String.format("{\n" + " \"query\": \"%s\"\n" + "}", query); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java index 49ef7c583e..fbe1e378e2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java @@ -7,6 +7,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; @@ -18,6 +19,7 @@ import org.junit.Test; import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.SQLIntegTestCase; @@ -215,4 +217,52 @@ public void testQueryWithoutFrom() { assertEquals(1, response.getInt("total")); assertEquals(1, response.getJSONArray("datarows").getJSONArray(0).getInt(0)); } + + @Test + public void testAlias() throws Exception { + String indexName = Index.ONLINE.getName(); + String aliasName = "alias_ONLINE"; + String filterQuery = "{\n" + " \"term\": {\n" + " \"107\": 72 \n" + " }\n" + "}"; + + // Execute the SQL query with filter + String selectQuery = "SELECT * FROM " + TEST_INDEX_ONLINE; + JSONObject initialResponse = + new JSONObject(executeFetchQuery(selectQuery, 10, "jdbc", filterQuery)); + assertEquals(initialResponse.getInt("size"), 10); + + // Create an alias + String createAliasQuery = + String.format( + "{ \"actions\": [ { \"add\": { \"index\": \"%s\", \"alias\": \"%s\" } } ] }", + indexName, aliasName); + Request createAliasRequest = new Request("POST", "/_aliases"); + createAliasRequest.setJsonEntity(createAliasQuery); + JSONObject aliasResponse = new JSONObject(executeRequest(createAliasRequest)); + + // Assert that alias creation was acknowledged + assertTrue(aliasResponse.getBoolean("acknowledged")); + + // Query using the alias + String aliasSelectQuery = String.format("SELECT * FROM %s", aliasName); + JSONObject aliasQueryResponse = new JSONObject(executeFetchQuery(aliasSelectQuery, 4, "jdbc")); + assertEquals(4, aliasQueryResponse.getInt("size")); + + // Query using the alias with filter + JSONObject aliasFilteredResponse = + new JSONObject(executeFetchQuery(aliasSelectQuery, 4, "jdbc", filterQuery)); + assertEquals(aliasFilteredResponse.getInt("size"), 4); + } + + private String executeFetchQuery(String query, int fetchSize, String requestType, String filter) + throws IOException { + String endpoint = "/_plugins/_sql?format=" + requestType; + String requestBody = makeRequest(query, fetchSize, filter); + + Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(requestBody); + + Response response = client().performRequest(sqlRequest); + String responseString = getResponseBody(response, true); + return responseString; + } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index 1dece1488a..6fd9c73375 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -26,6 +26,8 @@ import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.opensearch.action.admin.indices.alias.get.GetAliasesResponse; import org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.opensearch.action.search.ClearScrollResponse; @@ -164,7 +166,11 @@ private void populateResultSetFromDefaultCursor(DefaultCursor cursor) { private void loadFromEsState(Query query) { String indexName = fetchIndexName(query); String[] fieldNames = fetchFieldsAsArray(query); - + GetAliasesResponse getAliasesResponse = + client.admin().indices().getAliases(new GetAliasesRequest(indexName)).actionGet(); + if (getAliasesResponse != null && !getAliasesResponse.getAliases().isEmpty()) { + indexName = getAliasesResponse.getAliases().keySet().iterator().next(); + } // Reset boolean in the case of JOIN query where multiple calls to loadFromEsState() are made selectAll = isSimpleQuerySelectAll(query) || isJoinQuerySelectAll(query, fieldNames); From f6ca54cce5d8a4e8284c9822798077e72c84a66f Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 23 Oct 2024 12:34:48 -0700 Subject: [PATCH 24/32] [Manual Backport] Backport #2981 to 2.x (#3101) * Adding PIT for pagination queries in new SQL engine code paths * Fix legacy code using scroll API instead of PIT for batch physical operator * Fix local debugger issue * Refactor integ-tests data for PIT and fix unit tests * Address feedback comments * Adding test coverage --------- Signed-off-by: Manasvini B S Signed-off-by: Simeon Widdis Co-authored-by: Manasvini B Suryanarayana --- .../org/opensearch/sql/legacy/CursorIT.java | 2 +- .../org/opensearch/sql/ppl/StandaloneIT.java | 7 +- .../sql/sql/PaginationFilterIT.java | 35 +- .../sql/sql/StandalonePaginationIT.java | 1 + .../ppl/explain_filter_agg_push.json | 2 +- .../ppl/explain_filter_push.json | 2 +- .../ppl/explain_limit_push.json | 2 +- .../expectedOutput/ppl/explain_output.json | 2 +- .../expectedOutput/ppl/explain_sort_push.json | 2 +- .../executor/format/SelectResultSet.java | 2 +- .../query/planner/logical/node/TableScan.java | 7 + .../query/planner/physical/node/Paginate.java | 144 ++++++++ .../node/{scroll => }/SearchHitRow.java | 6 +- .../node/pointInTime/PointInTime.java | 73 ++++ .../planner/physical/node/scroll/Scroll.java | 157 +-------- .../node/scroll/SearchHitRowTest.java | 1 + .../unittest/planner/QueryPlannerTest.java | 2 + .../opensearch/client/OpenSearchClient.java | 17 + .../client/OpenSearchNodeClient.java | 59 +++- .../client/OpenSearchRestClient.java | 56 ++- .../request/OpenSearchQueryRequest.java | 170 ++++++++- .../request/OpenSearchRequestBuilder.java | 71 +++- .../opensearch/storage/OpenSearchIndex.java | 4 +- .../storage/scan/OpenSearchIndexScan.java | 16 +- .../client/OpenSearchNodeClientTest.java | 69 +++- .../client/OpenSearchRestClientTest.java | 62 ++++ .../OpenSearchExecutionEngineTest.java | 6 +- .../OpenSearchExecutionProtectorTest.java | 11 +- .../request/OpenSearchQueryRequestTest.java | 323 ++++++++++++++++- .../request/OpenSearchRequestBuilderTest.java | 324 ++++++++++++++++-- .../storage/OpenSearchIndexTest.java | 18 +- .../OpenSearchIndexScanPaginationTest.java | 11 +- .../storage/scan/OpenSearchIndexScanTest.java | 108 ++++-- 33 files changed, 1470 insertions(+), 302 deletions(-) create mode 100644 legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java rename legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/{scroll => }/SearchHitRow.java (97%) create mode 100644 legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index 4ee99198df..90f72cec40 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -182,7 +182,7 @@ public void validTotalResultWithAndWithoutPaginationOrderBy() throws IOException String selectQuery = StringUtils.format( "SELECT firstname, state FROM %s ORDER BY balance DESC ", TEST_INDEX_ACCOUNT); - verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 26, false); + verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 25, false); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index f81e1b6615..66f85b0754 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -149,8 +149,11 @@ public void onFailure(Exception e) { private Settings defaultSettings() { return new Settings() { - private final Map defaultSettings = - new ImmutableMap.Builder().put(Key.QUERY_SIZE_LIMIT, 200).build(); + private final Map defaultSettings = + new ImmutableMap.Builder() + .put(Key.QUERY_SIZE_LIMIT, 200) + .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true) + .build(); @Override public T getSettingValue(Key key) { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java index 038596cf57..9a945ec86f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java @@ -34,25 +34,30 @@ public class PaginationFilterIT extends SQLIntegTestCase { */ private static final Map STATEMENT_TO_NUM_OF_PAGES = Map.of( - "SELECT * FROM " + TestsConstants.TEST_INDEX_ACCOUNT, 1000, + "SELECT * FROM " + TestsConstants.TEST_INDEX_ACCOUNT, + 1000, "SELECT * FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " WHERE match(address, 'street')", - 385, + 385, "SELECT * FROM " - + TestsConstants.TEST_INDEX_ACCOUNT - + " WHERE match(address, 'street') AND match(city, 'Ola')", - 1, + + TestsConstants.TEST_INDEX_ACCOUNT + + " WHERE match(address, 'street') AND match(city, 'Ola')", + 1, "SELECT firstname, lastname, highlight(address) FROM " - + TestsConstants.TEST_INDEX_ACCOUNT - + " WHERE match(address, 'street') AND match(state, 'OH')", - 5, + + TestsConstants.TEST_INDEX_ACCOUNT + + " WHERE match(address, 'street') AND match(state, 'OH')", + 5, "SELECT firstname, lastname, highlight('*') FROM " - + TestsConstants.TEST_INDEX_ACCOUNT - + " WHERE match(address, 'street') AND match(state, 'OH')", - 5, - "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE true", 60, - "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id=10", 1, - "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id + 5=15", 1, - "SELECT * FROM " + TestsConstants.TEST_INDEX_BANK, 7); + + TestsConstants.TEST_INDEX_ACCOUNT + + " WHERE match(address, 'street') AND match(state, 'OH')", + 5, + "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE true", + 60, + "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id=10", + 1, + "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id + 5=15", + 1, + "SELECT * FROM " + TestsConstants.TEST_INDEX_BANK, + 7); private final String sqlStatement; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java index e884734c96..698e185abb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java @@ -166,6 +166,7 @@ private Settings defaultSettings() { new ImmutableMap.Builder() .put(Key.QUERY_SIZE_LIMIT, 200) .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1)) + .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true) .build(); @Override diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json index 568b397f07..8035822357 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json @@ -8,7 +8,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone\u003dfalse)" + "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json index 0e7087aa1f..3e92a17b97 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json @@ -8,7 +8,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"range\":{\"balance\":{\"from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone\u003dfalse)" + "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"range\":{\"balance\":{\"from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json index 51a627ea4d..0a0b58f17d 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json @@ -16,7 +16,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json index 8d45714283..bd7310810e 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json @@ -31,7 +31,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone\u003dfalse)" + "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json index af2a57e536..e2630e24f9 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json @@ -8,7 +8,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone\u003dfalse)" + "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index 6fd9c73375..2b9bdc349a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -573,7 +573,7 @@ private void populateDefaultCursor(DefaultCursor cursor) { Integer limit = cursor.getLimit(); long rowsLeft = rowsLeft(cursor.getFetchSize(), cursor.getLimit()); if (rowsLeft <= 0) { - // close the cursor + // Delete Point In Time ID if (LocalClusterState.state().getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) { String pitId = cursor.getPitId(); PointInTimeHandler pit = new PointInTimeHandlerImpl(client, pitId); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/TableScan.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/TableScan.java index 16af199ed7..59e6f27216 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/TableScan.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/TableScan.java @@ -5,11 +5,15 @@ package org.opensearch.sql.legacy.query.planner.logical.node; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.util.Map; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder; import org.opensearch.sql.legacy.query.planner.core.PlanNode; import org.opensearch.sql.legacy.query.planner.logical.LogicalOperator; import org.opensearch.sql.legacy.query.planner.physical.PhysicalOperator; +import org.opensearch.sql.legacy.query.planner.physical.node.pointInTime.PointInTime; import org.opensearch.sql.legacy.query.planner.physical.node.scroll.Scroll; /** Table scan */ @@ -33,6 +37,9 @@ public PlanNode[] children() { @Override public PhysicalOperator[] toPhysical(Map> optimalOps) { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + return new PhysicalOperator[] {new PointInTime(request, pageSize)}; + } return new PhysicalOperator[] {new Scroll(request, pageSize)}; } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java new file mode 100644 index 0000000000..5bf31bb691 --- /dev/null +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java @@ -0,0 +1,144 @@ +package org.opensearch.sql.legacy.query.planner.physical.node; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Objects; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Client; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.Strings; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.search.SearchHit; +import org.opensearch.sql.legacy.domain.Where; +import org.opensearch.sql.legacy.exception.SqlParseException; +import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import org.opensearch.sql.legacy.query.maker.QueryMaker; +import org.opensearch.sql.legacy.query.planner.core.ExecuteParams; +import org.opensearch.sql.legacy.query.planner.core.PlanNode; +import org.opensearch.sql.legacy.query.planner.physical.Row; +import org.opensearch.sql.legacy.query.planner.physical.estimation.Cost; +import org.opensearch.sql.legacy.query.planner.resource.ResourceManager; + +public abstract class Paginate extends BatchPhysicalOperator { + + /** Request to submit to OpenSearch to scan over */ + protected final TableInJoinRequestBuilder request; + + protected final int pageSize; + + protected Client client; + + protected SearchResponse searchResponse; + + protected Integer timeout; + + protected ResourceManager resourceMgr; + + public Paginate(TableInJoinRequestBuilder request, int pageSize) { + this.request = request; + this.pageSize = pageSize; + } + + @Override + public PlanNode[] children() { + return new PlanNode[0]; + } + + @Override + public Cost estimate() { + return new Cost(); + } + + @Override + public void open(ExecuteParams params) throws Exception { + super.open(params); + client = params.get(ExecuteParams.ExecuteParamType.CLIENT); + timeout = params.get(ExecuteParams.ExecuteParamType.TIMEOUT); + resourceMgr = params.get(ExecuteParams.ExecuteParamType.RESOURCE_MANAGER); + + Object filter = params.get(ExecuteParams.ExecuteParamType.EXTRA_QUERY_FILTER); + if (filter instanceof BoolQueryBuilder) { + request + .getRequestBuilder() + .setQuery(generateNewQueryWithExtraFilter((BoolQueryBuilder) filter)); + + if (LOG.isDebugEnabled()) { + LOG.debug( + "Received extra query filter, re-build query: {}", + Strings.toString( + XContentType.JSON, request.getRequestBuilder().request().source(), true, true)); + } + } + } + + @Override + protected Collection> prefetch() { + Objects.requireNonNull(client, "Client connection is not ready"); + Objects.requireNonNull(resourceMgr, "ResourceManager is not set"); + Objects.requireNonNull(timeout, "Time out is not set"); + + if (searchResponse == null) { + loadFirstBatch(); + updateMetaResult(); + } else { + loadNextBatch(); + } + return wrapRowForCurrentBatch(); + } + + protected abstract void loadFirstBatch(); + + protected abstract void loadNextBatch(); + + /** + * Extra filter pushed down from upstream. Re-parse WHERE clause with extra filter because + * OpenSearch RequestBuilder doesn't allow QueryBuilder inside be changed after added. + */ + protected QueryBuilder generateNewQueryWithExtraFilter(BoolQueryBuilder filter) + throws SqlParseException { + Where where = request.getOriginalSelect().getWhere(); + BoolQueryBuilder newQuery; + if (where != null) { + newQuery = QueryMaker.explain(where, false); + newQuery.must(filter); + } else { + newQuery = filter; + } + return newQuery; + } + + protected void updateMetaResult() { + resourceMgr.getMetaResult().addTotalNumOfShards(searchResponse.getTotalShards()); + resourceMgr.getMetaResult().addSuccessfulShards(searchResponse.getSuccessfulShards()); + resourceMgr.getMetaResult().addFailedShards(searchResponse.getFailedShards()); + resourceMgr.getMetaResult().updateTimeOut(searchResponse.isTimedOut()); + } + + @SuppressWarnings("unchecked") + protected Collection> wrapRowForCurrentBatch() { + SearchHit[] hits = searchResponse.getHits().getHits(); + Row[] rows = new Row[hits.length]; + for (int i = 0; i < hits.length; i++) { + rows[i] = new SearchHitRow(hits[i], request.getAlias()); + } + return Arrays.asList(rows); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [ " + describeTable() + ", pageSize=" + pageSize + " ]"; + } + + protected String describeTable() { + return request.getOriginalSelect().getFrom().get(0).getIndex() + " as " + request.getAlias(); + } + + /********************************************* + * Getters for Explain + *********************************************/ + + public String getRequest() { + return Strings.toString(XContentType.JSON, request.getRequestBuilder().request().source()); + } +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/SearchHitRow.java similarity index 97% rename from legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java rename to legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/SearchHitRow.java index a859a7c88c..a69c912da5 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/SearchHitRow.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.legacy.query.planner.physical.node.scroll; +package org.opensearch.sql.legacy.query.planner.physical.node; import com.google.common.base.Strings; import java.util.HashMap; @@ -32,7 +32,7 @@ * retain() in Project | {"firstName": "Allen", "age": 30 } | "" | retain("e.name.first", "e.age") * ---------------------------------------------------------------------------------------------------------------------- */ -class SearchHitRow implements Row { +public class SearchHitRow implements Row { /** Native OpenSearch data object for each row */ private final SearchHit hit; @@ -43,7 +43,7 @@ class SearchHitRow implements Row { /** Table alias owned the row. Empty if this row comes from combination of two other rows */ private final String tableAlias; - SearchHitRow(SearchHit hit, String tableAlias) { + public SearchHitRow(SearchHit hit, String tableAlias) { this.hit = hit; this.source = hit.getSourceAsMap(); this.tableAlias = tableAlias; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java new file mode 100644 index 0000000000..9ddbde2d29 --- /dev/null +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java @@ -0,0 +1,73 @@ +package org.opensearch.sql.legacy.query.planner.physical.node.pointInTime; + +import org.opensearch.common.unit.TimeValue; +import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; +import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import org.opensearch.sql.legacy.query.planner.physical.node.Paginate; + +/** OpenSearch Search API with Point in time as physical implementation of TableScan */ +public class PointInTime extends Paginate { + + private String pitId; + private PointInTimeHandlerImpl pit; + + public PointInTime(TableInJoinRequestBuilder request, int pageSize) { + super(request, pageSize); + } + + @Override + public void close() { + if (searchResponse != null) { + LOG.debug("Closing Point In Time (PIT) context"); + // Delete the Point In Time context + pit.delete(); + searchResponse = null; + } else { + LOG.debug("PIT context is already closed or was never opened"); + } + } + + @Override + protected void loadFirstBatch() { + // Create PIT and set to request object + pit = new PointInTimeHandlerImpl(client, request.getOriginalSelect().getIndexArr()); + pit.create(); + pitId = pit.getPitId(); + + LOG.info("Loading first batch of response using Point In Time"); + searchResponse = + request + .getRequestBuilder() + .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC) + .setSize(pageSize) + .setTimeout(TimeValue.timeValueSeconds(timeout)) + .setPointInTime(new PointInTimeBuilder(pitId)) + .get(); + } + + @Override + protected void loadNextBatch() { + // Add PIT with search after to fetch next batch of data + if (searchResponse.getHits().getHits() != null + && searchResponse.getHits().getHits().length > 0) { + Object[] sortValues = + searchResponse + .getHits() + .getHits()[searchResponse.getHits().getHits().length - 1] + .getSortValues(); + + LOG.info("Loading next batch of response using Point In Time. - " + pitId); + searchResponse = + request + .getRequestBuilder() + .setSize(pageSize) + .setTimeout(TimeValue.timeValueSeconds(timeout)) + .setPointInTime(new PointInTimeBuilder(pitId)) + .searchAfter(sortValues) + .get(); + } + } +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java index 40e9860886..5019e9cde8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java @@ -5,138 +5,38 @@ package org.opensearch.sql.legacy.query.planner.physical.node.scroll; -import java.util.Arrays; -import java.util.Collection; -import java.util.Objects; import org.opensearch.action.search.ClearScrollResponse; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.Client; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.common.Strings; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.search.SearchHit; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortOrder; -import org.opensearch.sql.legacy.domain.Where; -import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder; -import org.opensearch.sql.legacy.query.maker.QueryMaker; -import org.opensearch.sql.legacy.query.planner.core.ExecuteParams; -import org.opensearch.sql.legacy.query.planner.core.PlanNode; -import org.opensearch.sql.legacy.query.planner.physical.Row; -import org.opensearch.sql.legacy.query.planner.physical.estimation.Cost; -import org.opensearch.sql.legacy.query.planner.physical.node.BatchPhysicalOperator; -import org.opensearch.sql.legacy.query.planner.resource.ResourceManager; +import org.opensearch.sql.legacy.query.planner.physical.node.Paginate; /** OpenSearch Scroll API as physical implementation of TableScan */ -public class Scroll extends BatchPhysicalOperator { - - /** Request to submit to OpenSearch to scroll over */ - private final TableInJoinRequestBuilder request; - - /** Page size to scroll over index */ - private final int pageSize; - - /** Client connection to ElasticSearch */ - private Client client; - - /** Currently undergoing Scroll */ - private SearchResponse scrollResponse; - - /** Time out */ - private Integer timeout; - - /** Resource monitor manager */ - private ResourceManager resourceMgr; +public class Scroll extends Paginate { public Scroll(TableInJoinRequestBuilder request, int pageSize) { - this.request = request; - this.pageSize = pageSize; - } - - @Override - public PlanNode[] children() { - return new PlanNode[0]; - } - - @Override - public Cost estimate() { - return new Cost(); - } - - @Override - public void open(ExecuteParams params) throws Exception { - super.open(params); - client = params.get(ExecuteParams.ExecuteParamType.CLIENT); - timeout = params.get(ExecuteParams.ExecuteParamType.TIMEOUT); - resourceMgr = params.get(ExecuteParams.ExecuteParamType.RESOURCE_MANAGER); - - Object filter = params.get(ExecuteParams.ExecuteParamType.EXTRA_QUERY_FILTER); - if (filter instanceof BoolQueryBuilder) { - request - .getRequestBuilder() - .setQuery(generateNewQueryWithExtraFilter((BoolQueryBuilder) filter)); - - if (LOG.isDebugEnabled()) { - LOG.debug( - "Received extra query filter, re-build query: {}", - Strings.toString( - XContentType.JSON, request.getRequestBuilder().request().source(), true, true)); - } - } + super(request, pageSize); } @Override public void close() { - if (scrollResponse != null) { + if (searchResponse != null) { LOG.debug("Closing all scroll resources"); ClearScrollResponse clearScrollResponse = - client.prepareClearScroll().addScrollId(scrollResponse.getScrollId()).get(); + client.prepareClearScroll().addScrollId(searchResponse.getScrollId()).get(); if (!clearScrollResponse.isSucceeded()) { LOG.warn("Failed to close scroll: {}", clearScrollResponse.status()); } - scrollResponse = null; + searchResponse = null; } else { LOG.debug("Scroll already be closed"); } } @Override - protected Collection> prefetch() { - Objects.requireNonNull(client, "Client connection is not ready"); - Objects.requireNonNull(resourceMgr, "ResourceManager is not set"); - Objects.requireNonNull(timeout, "Time out is not set"); - - if (scrollResponse == null) { - loadFirstBatch(); - updateMetaResult(); - } else { - loadNextBatchByScrollId(); - } - return wrapRowForCurrentBatch(); - } - - /** - * Extra filter pushed down from upstream. Re-parse WHERE clause with extra filter because - * OpenSearch RequestBuilder doesn't allow QueryBuilder inside be changed after added. - */ - private QueryBuilder generateNewQueryWithExtraFilter(BoolQueryBuilder filter) - throws SqlParseException { - Where where = request.getOriginalSelect().getWhere(); - BoolQueryBuilder newQuery; - if (where != null) { - newQuery = QueryMaker.explain(where, false); - newQuery.must(filter); - } else { - newQuery = filter; - } - return newQuery; - } - - private void loadFirstBatch() { - scrollResponse = + protected void loadFirstBatch() { + searchResponse = request .getRequestBuilder() .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC) @@ -145,45 +45,12 @@ private void loadFirstBatch() { .get(); } - private void updateMetaResult() { - resourceMgr.getMetaResult().addTotalNumOfShards(scrollResponse.getTotalShards()); - resourceMgr.getMetaResult().addSuccessfulShards(scrollResponse.getSuccessfulShards()); - resourceMgr.getMetaResult().addFailedShards(scrollResponse.getFailedShards()); - resourceMgr.getMetaResult().updateTimeOut(scrollResponse.isTimedOut()); - } - - private void loadNextBatchByScrollId() { - scrollResponse = + @Override + protected void loadNextBatch() { + searchResponse = client - .prepareSearchScroll(scrollResponse.getScrollId()) + .prepareSearchScroll(searchResponse.getScrollId()) .setScroll(TimeValue.timeValueSeconds(timeout)) .get(); } - - @SuppressWarnings("unchecked") - private Collection> wrapRowForCurrentBatch() { - SearchHit[] hits = scrollResponse.getHits().getHits(); - Row[] rows = new Row[hits.length]; - for (int i = 0; i < hits.length; i++) { - rows[i] = new SearchHitRow(hits[i], request.getAlias()); - } - return Arrays.asList(rows); - } - - @Override - public String toString() { - return "Scroll [ " + describeTable() + ", pageSize=" + pageSize + " ]"; - } - - private String describeTable() { - return request.getOriginalSelect().getFrom().get(0).getIndex() + " as " + request.getAlias(); - } - - /********************************************* - * Getters for Explain - *********************************************/ - - public String getRequest() { - return Strings.toString(XContentType.JSON, request.getRequestBuilder().request().source()); - } } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java index dd0fc626c0..f7d2030b0c 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.search.SearchHit; +import org.opensearch.sql.legacy.query.planner.physical.node.SearchHitRow; public class SearchHitRowTest { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java index 521b225893..6ff907ba30 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.domain.JoinSelect; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; @@ -104,6 +105,7 @@ public void init() { // to mock. // In this case, default value in Setting will be returned all the time. doReturn(emptyList()).when(settings).getSettings(); + doReturn(false).when(settings).getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER); LocalClusterState.state().setPluginSettings(settings); ActionFuture mockFuture = mock(ActionFuture.class); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 0a9cc67993..cdc3d4462f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -7,6 +7,8 @@ import java.util.List; import java.util.Map; +import org.opensearch.action.search.CreatePitRequest; +import org.opensearch.action.search.DeletePitRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; @@ -89,4 +91,19 @@ public interface OpenSearchClient { void schedule(Runnable task); NodeClient getNodeClient(); + + /** + * Create PIT for given indices + * + * @param createPitRequest Create Point In Time request + * @return PitId + */ + String createPit(CreatePitRequest createPitRequest); + + /** + * Delete PIT + * + * @param deletePitRequest Delete Point In Time request + */ + void deletePit(DeletePitRequest deletePitRequest); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 993e092534..7a9487ef6a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -11,6 +11,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -21,13 +22,16 @@ import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.action.search.*; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; +import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; /** OpenSearch connection by node client. */ @@ -155,20 +159,32 @@ public List indices() { */ @Override public Map meta() { - return ImmutableMap.of(META_CLUSTER_NAME, client.settings().get("cluster.name", "opensearch")); + return ImmutableMap.of( + META_CLUSTER_NAME, + client.settings().get("cluster.name", "opensearch"), + "plugins.sql.pagination.api", + client.settings().get("plugins.sql.pagination.api", "true")); } @Override public void cleanup(OpenSearchRequest request) { - request.clean( - scrollId -> { - try { - client.prepareClearScroll().addScrollId(scrollId).get(); - } catch (Exception e) { - throw new IllegalStateException( - "Failed to clean up resources for search request " + request, e); - } - }); + if (request instanceof OpenSearchScrollRequest) { + request.clean( + scrollId -> { + try { + client.prepareClearScroll().addScrollId(scrollId).get(); + } catch (Exception e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } + }); + } else { + request.clean( + pitId -> { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + deletePit(deletePitRequest); + }); + } } @Override @@ -181,4 +197,27 @@ public void schedule(Runnable task) { public NodeClient getNodeClient() { return client; } + + @Override + public String createPit(CreatePitRequest createPitRequest) { + ActionFuture execute = + this.client.execute(CreatePitAction.INSTANCE, createPitRequest); + try { + CreatePitResponse pitResponse = execute.get(); + return pitResponse.getId(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Error occurred while creating PIT for new engine SQL query", e); + } + } + + @Override + public void deletePit(DeletePitRequest deletePitRequest) { + ActionFuture execute = + this.client.execute(DeletePitAction.INSTANCE, deletePitRequest); + try { + DeletePitResponse deletePitResponse = execute.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Error occurred while deleting PIT.", e); + } + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index b6106982a7..5cb6a69918 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -19,7 +19,7 @@ import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.opensearch.action.search.ClearScrollRequest; +import org.opensearch.action.search.*; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; import org.opensearch.client.indices.CreateIndexRequest; @@ -32,6 +32,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; +import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; /** @@ -166,6 +167,8 @@ public Map meta() { final Settings defaultSettings = client.cluster().getSettings(request, RequestOptions.DEFAULT).getDefaultSettings(); builder.put(META_CLUSTER_NAME, defaultSettings.get("cluster.name", "opensearch")); + builder.put( + "plugins.sql.pagination.api", defaultSettings.get("plugins.sql.pagination.api", "true")); return builder.build(); } catch (IOException e) { throw new IllegalStateException("Failed to get cluster meta info", e); @@ -174,17 +177,25 @@ public Map meta() { @Override public void cleanup(OpenSearchRequest request) { - request.clean( - scrollId -> { - try { - ClearScrollRequest clearRequest = new ClearScrollRequest(); - clearRequest.addScrollId(scrollId); - client.clearScroll(clearRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new IllegalStateException( - "Failed to clean up resources for search request " + request, e); - } - }); + if (request instanceof OpenSearchScrollRequest) { + request.clean( + scrollId -> { + try { + ClearScrollRequest clearRequest = new ClearScrollRequest(); + clearRequest.addScrollId(scrollId); + client.clearScroll(clearRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } + }); + } else { + request.clean( + pitId -> { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + deletePit(deletePitRequest); + }); + } } @Override @@ -196,4 +207,25 @@ public void schedule(Runnable task) { public NodeClient getNodeClient() { throw new UnsupportedOperationException("Unsupported method."); } + + @Override + public String createPit(CreatePitRequest createPitRequest) { + try { + CreatePitResponse createPitResponse = + client.createPit(createPitRequest, RequestOptions.DEFAULT); + return createPitResponse.getId(); + } catch (IOException e) { + throw new RuntimeException("Error occurred while creating PIT for new engine SQL query", e); + } + } + + @Override + public void deletePit(DeletePitRequest deletePitRequest) { + try { + DeletePitResponse deletePitResponse = + client.deletePit(deletePitRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException("Error occurred while creating PIT for new engine SQL query", e); + } + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index 6cf7fe49c2..fff252f3b4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -5,21 +5,35 @@ package org.opensearch.sql.opensearch.request; +import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; +import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; +import static org.opensearch.search.sort.SortOrder.ASC; + import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.function.Consumer; import java.util.function.Function; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.search.SearchScrollRequest; +import org.opensearch.action.search.*; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.search.SearchModule; +import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.OpenSearchResponse; +import org.opensearch.sql.opensearch.storage.OpenSearchIndex; +import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; /** * OpenSearch search request. This has to be stateful because it needs to: @@ -36,7 +50,7 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { private final IndexName indexName; /** Search request source builder. */ - private final SearchSourceBuilder sourceBuilder; + private SearchSourceBuilder sourceBuilder; /** OpenSearchExprValueFactory. */ @EqualsAndHashCode.Exclude @ToString.Exclude @@ -45,9 +59,19 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { /** List of includes expected in the response. */ @EqualsAndHashCode.Exclude @ToString.Exclude private final List includes; + @EqualsAndHashCode.Exclude private boolean needClean = true; + /** Indicate the search already done. */ private boolean searchDone = false; + private String pitId; + + private TimeValue cursorKeepAlive; + + private Object[] searchAfter; + + private SearchResponse searchResponse = null; + /** Constructor of OpenSearchQueryRequest. */ public OpenSearchQueryRequest( String indexName, int size, OpenSearchExprValueFactory factory, List includes) { @@ -78,35 +102,153 @@ public OpenSearchQueryRequest( this.includes = includes; } + /** Constructor of OpenSearchQueryRequest with PIT support. */ + public OpenSearchQueryRequest( + IndexName indexName, + SearchSourceBuilder sourceBuilder, + OpenSearchExprValueFactory factory, + List includes, + TimeValue cursorKeepAlive, + String pitId) { + this.indexName = indexName; + this.sourceBuilder = sourceBuilder; + this.exprValueFactory = factory; + this.includes = includes; + this.cursorKeepAlive = cursorKeepAlive; + this.pitId = pitId; + } + + /** + * Constructs OpenSearchQueryRequest from serialized representation. + * + * @param in stream to read data from. + * @param engine OpenSearchSqlEngine to get node-specific context. + * @throws IOException thrown if reading from input {@code in} fails. + */ + public OpenSearchQueryRequest(StreamInput in, OpenSearchStorageEngine engine) throws IOException { + // Deserialize the SearchSourceBuilder from the string representation + String sourceBuilderString = in.readString(); + + NamedXContentRegistry xContentRegistry = + new NamedXContentRegistry( + new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()); + XContentParser parser = + XContentType.JSON + .xContent() + .createParser(xContentRegistry, IGNORE_DEPRECATIONS, sourceBuilderString); + this.sourceBuilder = SearchSourceBuilder.fromXContent(parser); + + cursorKeepAlive = in.readTimeValue(); + pitId = in.readString(); + includes = in.readStringList(); + indexName = new IndexName(in); + + int length = in.readVInt(); + this.searchAfter = new Object[length]; + for (int i = 0; i < length; i++) { + this.searchAfter[i] = in.readGenericValue(); + } + + OpenSearchIndex index = (OpenSearchIndex) engine.getTable(null, indexName.toString()); + exprValueFactory = new OpenSearchExprValueFactory(index.getFieldOpenSearchTypes()); + } + @Override public OpenSearchResponse search( Function searchAction, Function scrollAction) { + if (this.pitId == null) { + // When SearchRequest doesn't contain PitId, fetch single page request + if (searchDone) { + return new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); + } else { + searchDone = true; + return new OpenSearchResponse( + searchAction.apply( + new SearchRequest().indices(indexName.getIndexNames()).source(sourceBuilder)), + exprValueFactory, + includes); + } + } else { + // Search with PIT instead of scroll API + return searchWithPIT(searchAction); + } + } + + public OpenSearchResponse searchWithPIT(Function searchAction) { + OpenSearchResponse openSearchResponse; if (searchDone) { - return new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); + openSearchResponse = new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); } else { - searchDone = true; - return new OpenSearchResponse( - searchAction.apply( - new SearchRequest().indices(indexName.getIndexNames()).source(sourceBuilder)), - exprValueFactory, - includes); + this.sourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(this.pitId)); + this.sourceBuilder.timeout(cursorKeepAlive); + // check for search after + if (searchAfter != null) { + this.sourceBuilder.searchAfter(searchAfter); + } + // Set sort field for search_after + if (this.sourceBuilder.sorts() == null) { + this.sourceBuilder.sort(DOC_FIELD_NAME, ASC); + } + SearchRequest searchRequest = new SearchRequest().source(this.sourceBuilder); + this.searchResponse = searchAction.apply(searchRequest); + + openSearchResponse = new OpenSearchResponse(this.searchResponse, exprValueFactory, includes); + + needClean = openSearchResponse.isEmpty(); + searchDone = openSearchResponse.isEmpty(); + SearchHit[] searchHits = this.searchResponse.getHits().getHits(); + if (searchHits != null && searchHits.length > 0) { + searchAfter = searchHits[searchHits.length - 1].getSortValues(); + this.sourceBuilder.searchAfter(searchAfter); + } } + return openSearchResponse; } @Override public void clean(Consumer cleanAction) { - // do nothing. + try { + // clean on the last page only, to prevent deleting the PitId in the middle of paging. + if (this.pitId != null && needClean) { + cleanAction.accept(this.pitId); + searchDone = true; + } + } finally { + this.pitId = null; + } } @Override public boolean hasAnotherBatch() { + if (this.pitId != null) { + return !needClean; + } return false; } @Override public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException( - "OpenSearchQueryRequest serialization " + "is not implemented."); + if (this.pitId != null) { + // Convert SearchSourceBuilder to XContent and write it as a string + out.writeString(sourceBuilder.toString()); + + out.writeTimeValue(sourceBuilder.timeout()); + out.writeString(sourceBuilder.pointInTimeBuilder().getId()); + out.writeStringCollection(includes); + indexName.writeTo(out); + + // Serialize the searchAfter array + if (searchAfter != null) { + out.writeVInt(searchAfter.length); + for (Object obj : searchAfter) { + out.writeGenericValue(obj); + } + } + } else { + // OpenSearch Query request without PIT for single page requests + throw new UnsupportedOperationException( + "OpenSearchQueryRequest serialization is not implemented."); + } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java index 1df3dcb183..6fa9b17697 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java @@ -26,6 +26,7 @@ import lombok.ToString; import org.apache.commons.lang3.tuple.Pair; import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.action.search.CreatePitRequest; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.InnerHitBuilder; @@ -39,9 +40,11 @@ import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -67,10 +70,13 @@ public class OpenSearchRequestBuilder { private int startFrom = 0; + private final Settings settings; + /** Constructor. */ public OpenSearchRequestBuilder( - int requestedTotalSize, OpenSearchExprValueFactory exprValueFactory) { + int requestedTotalSize, OpenSearchExprValueFactory exprValueFactory, Settings settings) { this.requestedTotalSize = requestedTotalSize; + this.settings = settings; this.sourceBuilder = new SearchSourceBuilder() .from(startFrom) @@ -82,18 +88,65 @@ public OpenSearchRequestBuilder( /** * Build DSL request. * - * @return query request or scroll request + * @return query request with PIT or scroll request */ public OpenSearchRequest build( - OpenSearchRequest.IndexName indexName, int maxResultWindow, TimeValue scrollTimeout) { + OpenSearchRequest.IndexName indexName, + int maxResultWindow, + TimeValue cursorKeepAlive, + OpenSearchClient client) { + if (this.settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) { + return buildRequestWithPit(indexName, maxResultWindow, cursorKeepAlive, client); + } else { + return buildRequestWithScroll(indexName, maxResultWindow, cursorKeepAlive); + } + } + + private OpenSearchRequest buildRequestWithPit( + OpenSearchRequest.IndexName indexName, + int maxResultWindow, + TimeValue cursorKeepAlive, + OpenSearchClient client) { + int size = requestedTotalSize; + FetchSourceContext fetchSource = this.sourceBuilder.fetchSource(); + List includes = fetchSource != null ? Arrays.asList(fetchSource.includes()) : List.of(); + + if (pageSize == null) { + if (startFrom + size > maxResultWindow) { + sourceBuilder.size(maxResultWindow - startFrom); + // Search with PIT request + String pitId = createPit(indexName, cursorKeepAlive, client); + return new OpenSearchQueryRequest( + indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); + } else { + sourceBuilder.from(startFrom); + sourceBuilder.size(requestedTotalSize); + // Search with non-Pit request + return new OpenSearchQueryRequest(indexName, sourceBuilder, exprValueFactory, includes); + } + } else { + if (startFrom != 0) { + throw new UnsupportedOperationException("Non-zero offset is not supported with pagination"); + } + sourceBuilder.size(pageSize); + // Search with PIT request + String pitId = createPit(indexName, cursorKeepAlive, client); + return new OpenSearchQueryRequest( + indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); + } + } + + private OpenSearchRequest buildRequestWithScroll( + OpenSearchRequest.IndexName indexName, int maxResultWindow, TimeValue cursorKeepAlive) { int size = requestedTotalSize; FetchSourceContext fetchSource = this.sourceBuilder.fetchSource(); List includes = fetchSource != null ? Arrays.asList(fetchSource.includes()) : List.of(); + if (pageSize == null) { if (startFrom + size > maxResultWindow) { sourceBuilder.size(maxResultWindow - startFrom); return new OpenSearchScrollRequest( - indexName, scrollTimeout, sourceBuilder, exprValueFactory, includes); + indexName, cursorKeepAlive, sourceBuilder, exprValueFactory, includes); } else { sourceBuilder.from(startFrom); sourceBuilder.size(requestedTotalSize); @@ -105,10 +158,18 @@ public OpenSearchRequest build( } sourceBuilder.size(pageSize); return new OpenSearchScrollRequest( - indexName, scrollTimeout, sourceBuilder, exprValueFactory, includes); + indexName, cursorKeepAlive, sourceBuilder, exprValueFactory, includes); } } + private String createPit( + OpenSearchRequest.IndexName indexName, TimeValue cursorKeepAlive, OpenSearchClient client) { + // Create PIT ID for request + CreatePitRequest createPitRequest = + new CreatePitRequest(cursorKeepAlive, false, indexName.getIndexNames()); + return client.createPit(createPitRequest); + } + boolean isBoolFilterQuery(QueryBuilder current) { return (current instanceof BoolQueryBuilder); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index c6afdb8511..a6fe83c8c4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -163,13 +163,13 @@ public TableScanBuilder createScanBuilder() { final int querySizeLimit = settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT); final TimeValue cursorKeepAlive = settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE); - var builder = new OpenSearchRequestBuilder(querySizeLimit, createExprValueFactory()); + var builder = new OpenSearchRequestBuilder(querySizeLimit, createExprValueFactory(), settings); Function createScanOperator = requestBuilder -> new OpenSearchIndexScan( client, requestBuilder.getMaxResponseSize(), - requestBuilder.build(indexName, getMaxResultWindow(), cursorKeepAlive)); + requestBuilder.build(indexName, getMaxResultWindow(), cursorKeepAlive, client)); return new OpenSearchIndexScanBuilder(builder, createScanOperator); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java index b1e4ccc463..b17773cb03 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java @@ -14,10 +14,12 @@ import lombok.ToString; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.NoCursorException; import org.opensearch.sql.executor.pagination.PlanSerializer; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; @@ -45,6 +47,8 @@ public class OpenSearchIndexScan extends TableScanOperator implements Serializab /** Search response for current batch. */ private Iterator iterator; + private Settings pluginSettings; + /** Creates index scan based on a provided OpenSearchRequestBuilder. */ public OpenSearchIndexScan( OpenSearchClient client, int maxResponseSize, OpenSearchRequest request) { @@ -121,12 +125,18 @@ public void readExternal(ObjectInput in) throws IOException { (OpenSearchStorageEngine) ((PlanSerializer.CursorDeserializationStream) in).resolveObject("engine"); + client = engine.getClient(); + boolean pointInTimeEnabled = + Boolean.parseBoolean( + client.meta().get(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER.getKeyValue())); try (BytesStreamInput bsi = new BytesStreamInput(requestStream)) { - request = new OpenSearchScrollRequest(bsi, engine); + if (pointInTimeEnabled) { + request = new OpenSearchQueryRequest(bsi, engine); + } else { + request = new OpenSearchScrollRequest(bsi, engine); + } } maxResponseSize = in.readInt(); - - client = engine.getClient(); } @Override diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 9da6e05e92..ba0fb85422 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -13,8 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -31,6 +30,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import lombok.SneakyThrows; import org.apache.commons.lang3.reflect.FieldUtils; @@ -51,15 +51,16 @@ import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.opensearch.action.search.ClearScrollRequestBuilder; -import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.*; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; @@ -74,6 +75,7 @@ import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.mapping.IndexMapping; +import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; @@ -393,6 +395,65 @@ void cleanup_rethrows_exception() { assertThrows(IllegalStateException.class, () -> client.cleanup(request)); } + @Test + @SneakyThrows + void cleanup_pit_request() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder(), + factory, + List.of(), + TimeValue.timeValueMinutes(1L), + "samplePitId"); + // Enforce cleaning by setting a private field. + FieldUtils.writeField(request, "needClean", true, true); + client.cleanup(request); + verify(nodeClient).execute(any(), any()); + } + + @Test + @SneakyThrows + void cleanup_pit_request_throw_exception() { + DeletePitRequest deletePitRequest = new DeletePitRequest("samplePitId"); + ActionFuture actionFuture = mock(ActionFuture.class); + when(actionFuture.get()).thenThrow(new ExecutionException("Execution failed", new Throwable())); + when(nodeClient.execute(eq(DeletePitAction.INSTANCE), any(DeletePitRequest.class))) + .thenReturn(actionFuture); + assertThrows(RuntimeException.class, () -> client.deletePit(deletePitRequest)); + } + + @Test + @SneakyThrows + void create_pit() { + CreatePitRequest createPitRequest = + new CreatePitRequest(TimeValue.timeValueMinutes(5), false, Strings.EMPTY_ARRAY); + ActionFuture actionFuture = mock(ActionFuture.class); + CreatePitResponse createPitResponse = mock(CreatePitResponse.class); + when(createPitResponse.getId()).thenReturn("samplePitId"); + when(actionFuture.get()).thenReturn(createPitResponse); + when(nodeClient.execute(eq(CreatePitAction.INSTANCE), any(CreatePitRequest.class))) + .thenReturn(actionFuture); + + String pitId = client.createPit(createPitRequest); + assertEquals("samplePitId", pitId); + + verify(nodeClient).execute(CreatePitAction.INSTANCE, createPitRequest); + verify(actionFuture).get(); + } + + @Test + @SneakyThrows + void create_pit_request_throw_exception() { + CreatePitRequest createPitRequest = + new CreatePitRequest(TimeValue.timeValueMinutes(5), false, Strings.EMPTY_ARRAY); + ActionFuture actionFuture = mock(ActionFuture.class); + when(actionFuture.get()).thenThrow(new ExecutionException("Execution failed", new Throwable())); + when(nodeClient.execute(eq(CreatePitAction.INSTANCE), any(CreatePitRequest.class))) + .thenReturn(actionFuture); + assertThrows(RuntimeException.class, () -> client.createPit(createPitRequest)); + } + @Test void get_indices() { AliasMetadata aliasMetadata = mock(AliasMetadata.class); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index b83313de07..82d9e74422 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -43,6 +43,8 @@ import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.action.search.CreatePitRequest; +import org.opensearch.action.search.CreatePitResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; @@ -56,6 +58,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; @@ -69,6 +72,7 @@ import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.mapping.IndexMapping; +import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; @@ -411,6 +415,64 @@ void cleanup_with_IOException() { assertThrows(IllegalStateException.class, () -> client.cleanup(request)); } + @Test + @SneakyThrows + void cleanup_pit_request() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder(), + factory, + List.of(), + TimeValue.timeValueMinutes(1L), + "samplePitId"); + // Enforce cleaning by setting a private field. + FieldUtils.writeField(request, "needClean", true, true); + client.cleanup(request); + verify(restClient).deletePit(any(), any()); + } + + @Test + @SneakyThrows + void cleanup_pit_request_throw_exception() { + when(restClient.deletePit(any(), any())).thenThrow(new IOException()); + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder(), + factory, + List.of(), + TimeValue.timeValueMinutes(1L), + "samplePitId"); + // Enforce cleaning by setting a private field. + FieldUtils.writeField(request, "needClean", true, true); + assertThrows(RuntimeException.class, () -> client.cleanup(request)); + } + + @Test + @SneakyThrows + void create_pit() { + CreatePitRequest createPitRequest = + new CreatePitRequest(TimeValue.timeValueMinutes(5), false, Strings.EMPTY_ARRAY); + CreatePitResponse createPitResponse = mock(CreatePitResponse.class); + when(createPitResponse.getId()).thenReturn("samplePitId"); + when(restClient.createPit(any(CreatePitRequest.class), any())).thenReturn(createPitResponse); + + String pitId = client.createPit(createPitRequest); + assertEquals("samplePitId", pitId); + + verify(restClient).createPit(createPitRequest, RequestOptions.DEFAULT); + } + + @Test + @SneakyThrows + void create_pit_request_throw_exception() { + CreatePitRequest createPitRequest = + new CreatePitRequest(TimeValue.timeValueMinutes(5), false, Strings.EMPTY_ARRAY); + when(restClient.createPit(any(), any())).thenThrow(new IOException()); + assertThrows(RuntimeException.class, () -> client.createPit(createPitRequest)); + } + @Test void get_indices() throws IOException { when(restClient.indices().get(any(GetIndexRequest.class), any(RequestOptions.class))) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java index 739b70b1b8..e5cf94eb86 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java @@ -174,18 +174,20 @@ void explain_successfully() { new OpenSearchExecutionEngine(client, protector, new PlanSerializer(null)); Settings settings = mock(Settings.class); when(settings.getSettingValue(SQL_CURSOR_KEEP_ALIVE)).thenReturn(TimeValue.timeValueMinutes(1)); + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); OpenSearchExprValueFactory exprValueFactory = mock(OpenSearchExprValueFactory.class); final var name = new OpenSearchRequest.IndexName("test"); final int defaultQuerySize = 100; final int maxResultWindow = 10000; - final var requestBuilder = new OpenSearchRequestBuilder(defaultQuerySize, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(defaultQuerySize, exprValueFactory, settings); PhysicalPlan plan = new OpenSearchIndexScan( mock(OpenSearchClient.class), maxResultWindow, requestBuilder.build( - name, maxResultWindow, settings.getSettingValue(SQL_CURSOR_KEEP_ALIVE))); + name, maxResultWindow, settings.getSettingValue(SQL_CURSOR_KEEP_ALIVE), client)); AtomicReference result = new AtomicReference<>(); executor.explain( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java index 5cd11c6cd4..da06c1eb66 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java @@ -8,9 +8,7 @@ import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.*; import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -91,6 +89,8 @@ public void setup() { @Test void test_protect_indexScan() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + String indexName = "test"; final int maxResultWindow = 10000; final int querySizeLimit = 200; @@ -114,11 +114,12 @@ void test_protect_indexScan() { final var name = new OpenSearchRequest.IndexName(indexName); final var request = - new OpenSearchRequestBuilder(querySizeLimit, exprValueFactory) + new OpenSearchRequestBuilder(querySizeLimit, exprValueFactory, settings) .build( name, maxResultWindow, - settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE)); + settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE), + client); assertEquals( PhysicalPlanDSL.project( PhysicalPlanDSL.limit( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java index d2bc5b0641..89b51207b5 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java @@ -5,37 +5,40 @@ package org.opensearch.sql.opensearch.request; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.opensearch.sql.opensearch.request.OpenSearchRequest.DEFAULT_QUERY_TIMEOUT; +import java.io.IOException; +import java.lang.reflect.Field; import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import lombok.SneakyThrows; import org.apache.lucene.search.TotalHits; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.FetchSourceContext; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.OpenSearchResponse; +import org.opensearch.sql.opensearch.storage.OpenSearchIndex; +import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; @ExtendWith(MockitoExtension.class) public class OpenSearchQueryRequestTest { @@ -64,6 +67,93 @@ public class OpenSearchQueryRequestTest { private final OpenSearchQueryRequest remoteRequest = new OpenSearchQueryRequest("ccs:test", 200, factory, List.of()); + @Mock private StreamOutput streamOutput; + @Mock private StreamInput streamInput; + @Mock private OpenSearchStorageEngine engine; + @Mock private PointInTimeBuilder pointInTimeBuilder; + + @InjectMocks private OpenSearchQueryRequest serializationRequest; + + private SearchSourceBuilder sourceBuilderForSerializer; + + @BeforeEach + void setup() { + sourceBuilderForSerializer = new SearchSourceBuilder(); + sourceBuilderForSerializer.pointInTimeBuilder(pointInTimeBuilder); + sourceBuilderForSerializer.timeout(TimeValue.timeValueSeconds(30)); + } + + @SneakyThrows + @Test + void testWriteTo() throws IOException { + when(pointInTimeBuilder.getId()).thenReturn("samplePITId"); + sourceBuilderForSerializer.searchAfter(new Object[] {"value1", 123}); + List includes = List.of("field1", "field2"); + serializationRequest = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilderForSerializer, + factory, + includes, + new TimeValue(1000), + "samplePITId"); + + Field searchAfterField = OpenSearchQueryRequest.class.getDeclaredField("searchAfter"); + searchAfterField.setAccessible(true); + searchAfterField.set(serializationRequest, new Object[] {"value1", 123}); + + serializationRequest.writeTo(streamOutput); + + String expectedJson = "{\"timeout\":\"30s\",\"search_after\":[\"value1\",123]}"; + verify(streamOutput).writeString(expectedJson); + verify(streamOutput).writeTimeValue(TimeValue.timeValueSeconds(30)); + verify(streamOutput).writeString("samplePITId"); + verify(streamOutput).writeStringCollection(includes); + + verify(streamOutput).writeVInt(2); + verify(streamOutput).writeGenericValue("value1"); + verify(streamOutput).writeGenericValue(123); + } + + @Test + void testWriteToWithoutSearchAfter() + throws IOException, NoSuchFieldException, IllegalAccessException { + when(pointInTimeBuilder.getId()).thenReturn("samplePITId"); + + List includes = List.of("field1", "field2"); + serializationRequest = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilderForSerializer, + factory, + includes, + new TimeValue(1000), + "samplePITId"); + + serializationRequest.writeTo(streamOutput); + verify(streamOutput).writeString("{\"timeout\":\"30s\"}"); + verify(streamOutput).writeTimeValue(TimeValue.timeValueSeconds(30)); + verify(streamOutput).writeString("samplePITId"); + verify(streamOutput).writeStringCollection(includes); + verify(streamOutput, never()).writeVInt(anyInt()); + verify(streamOutput, never()).writeGenericValue(any()); + } + + @Test + void testWriteToWithoutPIT() { + serializationRequest = new OpenSearchQueryRequest("test", 200, factory, List.of()); + + UnsupportedOperationException exception = + assertThrows( + UnsupportedOperationException.class, + () -> { + request.writeTo(streamOutput); + }); + + assertEquals( + "OpenSearchQueryRequest serialization is not implemented.", exception.getMessage()); + } + @Test void search() { OpenSearchQueryRequest request = @@ -81,6 +171,145 @@ void search() { verify(searchAction, times(1)).apply(any()); } + @Test + void search_with_pit() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + when(searchHit.getSortValues()).thenReturn(new String[] {"sortedValue"}); + when(sourceBuilder.sorts()).thenReturn(null); + + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + assertFalse(openSearchResponse.isEmpty()); + verify(searchAction, times(1)).apply(any()); + + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchResponse.getAggregations()).thenReturn(null); + when(searchHits.getHits()).thenReturn(null); + openSearchResponse = request.searchWithPIT(searchAction); + assertTrue(openSearchResponse.isEmpty()); + verify(searchAction, times(2)).apply(any()); + + openSearchResponse = request.searchWithPIT(searchAction); + assertTrue(openSearchResponse.isEmpty()); + } + + @Test + void search_with_pit_hits_null() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + when(sourceBuilder.sorts()).thenReturn(null); + + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + assertFalse(openSearchResponse.isEmpty()); + } + + @Test + void search_with_pit_hits_empty() { + SearchResponse searchResponse = mock(SearchResponse.class); + SearchHits searchHits = mock(SearchHits.class); + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {}); + when(sourceBuilder.sorts()).thenReturn(null); + + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + assertTrue(openSearchResponse.isEmpty()); + } + + @Test + void search_with_pit_null() { + SearchResponse searchResponse = mock(SearchResponse.class); + SearchHits searchHits = mock(SearchHits.class); + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "sample"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + + OpenSearchResponse openSearchResponse = request.search(searchAction, scrollAction); + assertFalse(openSearchResponse.isEmpty()); + } + + @Test + void has_another_batch() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "sample"); + assertFalse(request.hasAnotherBatch()); + } + + @Test + void has_another_batch_pid_null() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + null); + assertFalse(request.hasAnotherBatch()); + } + + @Test + void has_another_batch_need_clean() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + assertTrue(request.hasAnotherBatch()); + } + @Test void search_withoutContext() { OpenSearchQueryRequest request = @@ -121,6 +350,68 @@ void clean() { verify(cleanAction, never()).accept(any()); } + @Test + void testCleanConditionTrue() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(null); + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + + request.clean(cleanAction); + + verify(cleanAction, times(1)).accept("samplePid"); + assertTrue(request.isSearchDone()); + assertNull(request.getPitId()); + } + + @Test + void testCleanConditionFalse_needCleanFalse() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + + request.clean(cleanAction); + verify(cleanAction, never()).accept(anyString()); + assertFalse(request.isSearchDone()); + assertNull(request.getPitId()); + } + + @Test + void testCleanConditionFalse_pidNull() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + null); + + request.clean(cleanAction); + verify(cleanAction, never()).accept(anyString()); + assertFalse(request.isSearchDone()); + assertNull(request.getPitId()); + } + @Test void searchRequest() { request.getSourceBuilder().query(QueryBuilders.termQuery("name", "John")); @@ -159,6 +450,20 @@ void writeTo_unsupported() { UnsupportedOperationException.class, () -> request.writeTo(mock(StreamOutput.class))); } + @Test + void constructor_serialized() throws IOException { + StreamInput stream = mock(StreamInput.class); + OpenSearchStorageEngine engine = mock(OpenSearchStorageEngine.class); + when(stream.readString()).thenReturn("{}"); + when(stream.readStringArray()).thenReturn(new String[] {"sample"}); + OpenSearchIndex index = mock(OpenSearchIndex.class); + when(engine.getTable(null, "sample")).thenReturn(index); + when(stream.readVInt()).thenReturn(2); + when(stream.readGenericValue()).thenReturn("sampleSearchAfter"); + OpenSearchQueryRequest request = new OpenSearchQueryRequest(stream, engine); + assertNotNull(request); + } + private void assertSearchRequest(SearchRequest expected, OpenSearchQueryRequest request) { Function querySearch = searchRequest -> { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java index 742e76cbd0..bf87840b60 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java @@ -5,13 +5,10 @@ package org.opensearch.sql.opensearch.request; -import static org.junit.Assert.assertThrows; +import static org.junit.Assert.*; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.opensearch.index.query.QueryBuilders.matchAllQuery; -import static org.opensearch.index.query.QueryBuilders.nestedQuery; +import static org.mockito.Mockito.*; +import static org.opensearch.index.query.QueryBuilders.*; import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; import static org.opensearch.search.sort.SortOrder.ASC; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -25,21 +22,16 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.join.ScoreMode; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayNameGeneration; -import org.junit.jupiter.api.DisplayNameGenerator; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.search.CreatePitRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.query.InnerHitBuilder; -import org.opensearch.index.query.NestedQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.*; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.AggregationBuilder; @@ -47,13 +39,19 @@ import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.ScoreSortBuilder; import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; @@ -76,11 +74,18 @@ class OpenSearchRequestBuilderTest { @Mock private OpenSearchExprValueFactory exprValueFactory; + @Mock private OpenSearchClient client; + + @Mock private Settings settings; + private OpenSearchRequestBuilder requestBuilder; @BeforeEach void setup() { - requestBuilder = new OpenSearchRequestBuilder(DEFAULT_LIMIT, exprValueFactory); + requestBuilder = new OpenSearchRequestBuilder(DEFAULT_LIMIT, exprValueFactory, settings); + lenient() + .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); } @Test @@ -100,14 +105,148 @@ void build_query_request() { .trackScores(true), exprValueFactory, List.of()), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void build_query_request_push_down_size() { + Integer limit = 200; + Integer offset = 0; + requestBuilder.pushDownLimit(limit, offset); + requestBuilder.pushDownTrackedScore(true); + + assertNotNull( + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void build_PIT_request_with_correct_size() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId"); + Integer limit = 0; + Integer offset = 0; + requestBuilder.pushDownLimit(limit, offset); + requestBuilder.pushDownPageSize(2); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder().from(offset).size(2).timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of(), + TimeValue.timeValueMinutes(1), + "samplePITId"), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithPit_pageSizeNull_sizeGreaterThanMaxResultWindow() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId"); + Integer limit = 600; + Integer offset = 0; + int requestedTotalSize = 600; + requestBuilder = new OpenSearchRequestBuilder(requestedTotalSize, exprValueFactory, settings); + requestBuilder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder() + .from(offset) + .size(MAX_RESULT_WINDOW - offset) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of(), + TimeValue.timeValueMinutes(1), + "samplePITId"), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithPit_pageSizeNull_sizeLessThanMaxResultWindow() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + Integer limit = 400; + Integer offset = 0; + int requestedTotalSize = 400; + requestBuilder = new OpenSearchRequestBuilder(requestedTotalSize, exprValueFactory, settings); + requestBuilder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder() + .from(offset) + .size(requestedTotalSize) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of()), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithPit_pageSizeNotNull_startFromZero() { + int pageSize = 200; + int offset = 0; + int limit = 400; + requestBuilder.pushDownPageSize(pageSize); + requestBuilder.pushDownLimit(limit, offset); + when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId"); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder().from(offset).size(pageSize).timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of(), + TimeValue.timeValueMinutes(1), + "samplePITId"), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithPit_pageSizeNotNull_startFromNonZero() { + int pageSize = 200; + int offset = 100; + int limit = 400; + requestBuilder.pushDownPageSize(pageSize); + requestBuilder.pushDownLimit(limit, offset); + assertThrows( + UnsupportedOperationException.class, + () -> { + requestBuilder.build(indexName, 500, TimeValue.timeValueMinutes(1), client); + }); } @Test void build_scroll_request_with_correct_size() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); Integer limit = 800; Integer offset = 10; requestBuilder.pushDownLimit(limit, offset); + requestBuilder.getSourceBuilder().fetchSource("a", "b"); + + assertEquals( + new OpenSearchScrollRequest( + new OpenSearchRequest.IndexName("test"), + TimeValue.timeValueMinutes(1), + new SearchSourceBuilder() + .from(offset) + .size(MAX_RESULT_WINDOW - offset) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of()), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithScroll_pageSizeNull_sizeGreaterThanMaxResultWindow() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + Integer limit = 600; + Integer offset = 0; + int requestedTotalSize = 600; + requestBuilder = new OpenSearchRequestBuilder(requestedTotalSize, exprValueFactory, settings); + requestBuilder.pushDownLimit(limit, offset); assertEquals( new OpenSearchScrollRequest( @@ -119,7 +258,65 @@ void build_scroll_request_with_correct_size() { .timeout(DEFAULT_QUERY_TIMEOUT), exprValueFactory, List.of()), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithScroll_pageSizeNull_sizeLessThanMaxResultWindow() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + Integer limit = 400; + Integer offset = 0; + int requestedTotalSize = 400; + requestBuilder = new OpenSearchRequestBuilder(requestedTotalSize, exprValueFactory, settings); + requestBuilder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder() + .from(offset) + .size(requestedTotalSize) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of()), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithScroll_pageSizeNotNull_startFromZero() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + int pageSize = 200; + int offset = 0; + int limit = 400; + requestBuilder.pushDownPageSize(pageSize); + requestBuilder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchScrollRequest( + new OpenSearchRequest.IndexName("test"), + TimeValue.timeValueMinutes(1), + new SearchSourceBuilder() + .from(offset) + .size(MAX_RESULT_WINDOW - offset) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of()), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithScroll_pageSizeNotNull_startFromNonZero() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + int pageSize = 200; + int offset = 100; + int limit = 400; + requestBuilder.pushDownPageSize(pageSize); + requestBuilder.pushDownLimit(limit, offset); + assertThrows( + UnsupportedOperationException.class, + () -> { + requestBuilder.build(indexName, 500, TimeValue.timeValueMinutes(1), client); + }); } @Test @@ -127,7 +324,7 @@ void test_push_down_query() { QueryBuilder query = QueryBuilders.termQuery("intA", 1); requestBuilder.pushDownFilter(query); - var r = requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT); + var r = requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client); Function querySearch = searchRequest -> { assertEquals( @@ -203,6 +400,51 @@ void test_push_down_query_and_sort() { requestBuilder); } + @Test + void test_push_down_query_not_null() { + SearchSourceBuilder sourceBuilder = requestBuilder.getSourceBuilder(); + sourceBuilder.query(QueryBuilders.termQuery("name", "John")); + sourceBuilder.sort(DOC_FIELD_NAME, SortOrder.ASC); + + QueryBuilder query = QueryBuilders.termQuery("intA", 1); + requestBuilder.pushDownFilter(query); + + BoolQueryBuilder expectedQuery = + QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("name", "John")).filter(query); + + SearchSourceBuilder expectedSourceBuilder = + new SearchSourceBuilder() + .from(DEFAULT_OFFSET) + .size(DEFAULT_LIMIT) + .timeout(DEFAULT_QUERY_TIMEOUT) + .query(expectedQuery) + .sort(DOC_FIELD_NAME, SortOrder.ASC); + + assertSearchSourceBuilder(expectedSourceBuilder, requestBuilder); + } + + @Test + void test_push_down_query_with_bool_filter() { + BoolQueryBuilder initialBoolQuery = + QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("name", "John")); + + SearchSourceBuilder sourceBuilder = requestBuilder.getSourceBuilder(); + sourceBuilder.query(initialBoolQuery); + + QueryBuilder newQuery = QueryBuilders.termQuery("intA", 1); + requestBuilder.pushDownFilter(newQuery); + initialBoolQuery.filter(newQuery); + SearchSourceBuilder expectedSourceBuilder = + new SearchSourceBuilder() + .from(DEFAULT_OFFSET) + .size(DEFAULT_LIMIT) + .timeout(DEFAULT_QUERY_TIMEOUT) + .query(initialBoolQuery) + .sort(DOC_FIELD_NAME, SortOrder.ASC); + + assertSearchSourceBuilder(expectedSourceBuilder, requestBuilder); + } + void assertSearchSourceBuilder( SearchSourceBuilder expected, OpenSearchRequestBuilder requestBuilder) throws UnsupportedOperationException { @@ -220,7 +462,7 @@ void assertSearchSourceBuilder( throw new UnsupportedOperationException(); }; requestBuilder - .build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT) + .build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client) .search(querySearch, scrollSearch); } @@ -290,7 +532,7 @@ void test_push_down_project() { .fetchSource("intA", null), exprValueFactory, List.of("intA")), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); } @Test @@ -320,7 +562,7 @@ void test_push_down_project_limit() { .fetchSource("intA", null), exprValueFactory, List.of("intA")), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); } @Test @@ -350,7 +592,7 @@ void test_push_down_project_limit_and_offset() { .fetchSource("intA", null), exprValueFactory, List.of("intA")), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); } @Test @@ -377,7 +619,7 @@ void test_push_down_nested() { assertSearchSourceBuilder( new SearchSourceBuilder() - .query(QueryBuilders.boolQuery().filter(QueryBuilders.boolQuery().must(nestedQuery))) + .query(boolQuery().filter(boolQuery().must(nestedQuery))) .from(DEFAULT_OFFSET) .size(DEFAULT_LIMIT) .timeout(DEFAULT_QUERY_TIMEOUT), @@ -411,7 +653,7 @@ void test_push_down_multiple_nested_with_same_path() { true, new String[] {"message.info", "message.from"}, null))); assertSearchSourceBuilder( new SearchSourceBuilder() - .query(QueryBuilders.boolQuery().filter(QueryBuilders.boolQuery().must(nestedQuery))) + .query(boolQuery().filter(boolQuery().must(nestedQuery))) .from(DEFAULT_OFFSET) .size(DEFAULT_LIMIT) .timeout(DEFAULT_QUERY_TIMEOUT), @@ -444,9 +686,9 @@ void test_push_down_nested_with_filter() { assertSearchSourceBuilder( new SearchSourceBuilder() .query( - QueryBuilders.boolQuery() + boolQuery() .filter( - QueryBuilders.boolQuery() + boolQuery() .must(QueryBuilders.rangeQuery("myNum").gt(3)) .must(nestedQuery))) .from(DEFAULT_OFFSET) @@ -483,7 +725,7 @@ void testPushDownNestedWithNestedFilter() { assertSearchSourceBuilder( new SearchSourceBuilder() - .query(QueryBuilders.boolQuery().filter(QueryBuilders.boolQuery().must(filterQuery))) + .query(boolQuery().filter(boolQuery().must(filterQuery))) .from(DEFAULT_OFFSET) .size(DEFAULT_LIMIT) .timeout(DEFAULT_QUERY_TIMEOUT), @@ -507,6 +749,32 @@ void push_down_highlight_with_repeating_fields() { assertEquals("Duplicate field name in highlight", exception.getMessage()); } + @Test + void test_push_down_highlight_with_pre_tags() { + requestBuilder.pushDownHighlight( + "name", Map.of("pre_tags", new Literal("pre1", DataType.STRING))); + + SearchSourceBuilder sourceBuilder = requestBuilder.getSourceBuilder(); + assertNotNull(sourceBuilder.highlighter()); + assertEquals(1, sourceBuilder.highlighter().fields().size()); + HighlightBuilder.Field field = sourceBuilder.highlighter().fields().get(0); + assertEquals("name", field.name()); + assertEquals("pre1", field.preTags()[0]); + } + + @Test + void test_push_down_highlight_with_post_tags() { + requestBuilder.pushDownHighlight( + "name", Map.of("post_tags", new Literal("post1", DataType.STRING))); + + SearchSourceBuilder sourceBuilder = requestBuilder.getSourceBuilder(); + assertNotNull(sourceBuilder.highlighter()); + assertEquals(1, sourceBuilder.highlighter().fields().size()); + HighlightBuilder.Field field = sourceBuilder.highlighter().fields().get(0); + assertEquals("name", field.name()); + assertEquals("post1", field.postTags()[0]); + } + @Test void push_down_page_size() { requestBuilder.pushDownPageSize(3); @@ -521,7 +789,7 @@ void exception_when_non_zero_offset_and_page_size() { requestBuilder.pushDownLimit(300, 2); assertThrows( UnsupportedOperationException.class, - () -> requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + () -> requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 3ca566fac6..ef6b86c42a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -79,6 +79,9 @@ class OpenSearchIndexTest { @BeforeEach void setUp() { this.index = new OpenSearchIndex(client, settings, "test"); + lenient() + .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); } @Test @@ -198,10 +201,11 @@ void implementRelationOperatorOnly() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); LogicalPlan plan = index.createScanBuilder(); Integer maxResultWindow = index.getMaxResultWindow(); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory, settings); assertEquals( new OpenSearchIndexScan( - client, 200, requestBuilder.build(INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT)), + client, 200, requestBuilder.build(INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT, client)), index.implement(index.optimize(plan))); } @@ -211,10 +215,11 @@ void implementRelationOperatorWithOptimization() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); LogicalPlan plan = index.createScanBuilder(); Integer maxResultWindow = index.getMaxResultWindow(); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory, settings); assertEquals( new OpenSearchIndexScan( - client, 200, requestBuilder.build(INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT)), + client, 200, requestBuilder.build(INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT, client)), index.implement(plan)); } @@ -243,7 +248,8 @@ void implementOtherLogicalOperators() { include); Integer maxResultWindow = index.getMaxResultWindow(); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory, settings); assertEquals( PhysicalPlanDSL.project( PhysicalPlanDSL.dedupe( @@ -255,7 +261,7 @@ void implementOtherLogicalOperators() { client, QUERY_SIZE_LIMIT, requestBuilder.build( - INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT)), + INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT, client)), mappings), exclude), newEvalField), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java index 2085519b12..e6a17aceaf 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java @@ -56,6 +56,9 @@ void setup() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE)) .thenReturn(TimeValue.timeValueMinutes(1)); + lenient() + .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); } @Mock private OpenSearchClient client; @@ -69,12 +72,12 @@ void setup() { @Test void query_empty_result() { mockResponse(client); - var builder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory); + var builder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory, settings); try (var indexScan = new OpenSearchIndexScan( client, MAX_RESULT_WINDOW, - builder.build(INDEX_NAME, MAX_RESULT_WINDOW, SCROLL_TIMEOUT))) { + builder.build(INDEX_NAME, MAX_RESULT_WINDOW, SCROLL_TIMEOUT, client))) { indexScan.open(); assertFalse(indexScan.hasNext()); } @@ -96,13 +99,13 @@ void dont_serialize_if_no_cursor() { OpenSearchRequestBuilder builder = mock(); OpenSearchRequest request = mock(); OpenSearchResponse response = mock(); - when(builder.build(any(), anyInt(), any())).thenReturn(request); + when(builder.build(any(), anyInt(), any(), any())).thenReturn(request); when(client.search(any())).thenReturn(response); try (var indexScan = new OpenSearchIndexScan( client, MAX_RESULT_WINDOW, - builder.build(INDEX_NAME, MAX_RESULT_WINDOW, SCROLL_TIMEOUT))) { + builder.build(INDEX_NAME, MAX_RESULT_WINDOW, SCROLL_TIMEOUT, client))) { indexScan.open(); when(request.hasAnotherBatch()).thenReturn(false); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java index f813d8f551..e680c6b3a6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java @@ -5,24 +5,19 @@ package org.opensearch.sql.opensearch.storage.scan; +import static org.junit.Assert.assertNotNull; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; import static org.opensearch.search.sort.SortOrder.ASC; import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; +import java.io.*; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -52,6 +47,7 @@ import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.NoCursorException; @@ -77,6 +73,7 @@ class OpenSearchIndexScanTest { public static final int MAX_RESULT_WINDOW = 10000; public static final TimeValue CURSOR_KEEP_ALIVE = TimeValue.timeValueMinutes(1); @Mock private OpenSearchClient client; + @Mock private Settings settings; private final OpenSearchExprValueFactory exprValueFactory = new OpenSearchExprValueFactory( @@ -84,7 +81,11 @@ class OpenSearchIndexScanTest { "name", OpenSearchDataType.of(STRING), "department", OpenSearchDataType.of(STRING))); @BeforeEach - void setup() {} + void setup() { + lenient() + .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); + } @Test void explain() { @@ -144,6 +145,49 @@ void serialize(Integer numberOfIncludes) { } } + @SneakyThrows + @ParameterizedTest + @ValueSource(ints = {0, 150}) + void serialize_PIT(Integer numberOfIncludes) { + var searchSourceBuilder = new SearchSourceBuilder().size(4); + + var factory = mock(OpenSearchExprValueFactory.class); + var engine = mock(OpenSearchStorageEngine.class); + var index = mock(OpenSearchIndex.class); + when(engine.getClient()).thenReturn(client); + when(engine.getTable(any(), any())).thenReturn(index); + Map map = mock(Map.class); + when(map.get(any(String.class))).thenReturn("true"); + when(client.meta()).thenReturn(map); + var includes = + Stream.iterate(1, i -> i + 1) + .limit(numberOfIncludes) + .map(i -> "column" + i) + .collect(Collectors.toList()); + var request = + new OpenSearchQueryRequest( + INDEX_NAME, searchSourceBuilder, factory, includes, CURSOR_KEEP_ALIVE, "samplePitId"); + // make a response, so OpenSearchResponse::isEmpty would return true and unset needClean + var response = mock(SearchResponse.class); + when(response.getAggregations()).thenReturn(mock()); + var hits = mock(SearchHits.class); + when(response.getHits()).thenReturn(hits); + SearchHit hit = mock(SearchHit.class); + when(hit.getSortValues()).thenReturn(new String[] {"sample1"}); + when(hits.getHits()).thenReturn(new SearchHit[] {hit}); + request.search((req) -> response, null); + + try (var indexScan = new OpenSearchIndexScan(client, QUERY_SIZE, request)) { + var planSerializer = new PlanSerializer(engine); + var cursor = planSerializer.convertToCursor(indexScan); + var newPlan = planSerializer.convertToPlan(cursor.toString()); + assertNotNull(newPlan); + + verify(client).meta(); + verify(map).get(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER.getKeyValue()); + } + } + @SneakyThrows @Test void throws_io_exception_if_too_short() { @@ -172,10 +216,12 @@ void plan_for_serialization() { void query_empty_result() { mockResponse(client); final var name = new OpenSearchRequest.IndexName("test"); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory); + final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, QUERY_SIZE, requestBuilder.build(name, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE))) { + client, + QUERY_SIZE, + requestBuilder.build(name, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertFalse(indexScan.hasNext()); } @@ -190,10 +236,10 @@ void query_all_results_with_query() { employee(1, "John", "IT"), employee(2, "Smith", "HR"), employee(3, "Allen", "IT") }); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory); + final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, 10, requestBuilder.build(INDEX_NAME, 10000, CURSOR_KEEP_ALIVE))) { + client, 10, requestBuilder.build(INDEX_NAME, 10000, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -218,10 +264,10 @@ void query_all_results_with_scroll() { new ExprValue[] {employee(1, "John", "IT"), employee(2, "Smith", "HR")}, new ExprValue[] {employee(3, "Allen", "IT")}); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory); + final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, 10, requestBuilder.build(INDEX_NAME, 10000, CURSOR_KEEP_ALIVE))) { + client, 10, requestBuilder.build(INDEX_NAME, 10000, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -248,10 +294,12 @@ void query_some_results_with_query() { }); final int limit = 3; - OpenSearchRequestBuilder builder = new OpenSearchRequestBuilder(0, exprValueFactory); + OpenSearchRequestBuilder builder = new OpenSearchRequestBuilder(0, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, limit, builder.build(INDEX_NAME, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE))) { + client, + limit, + builder.build(INDEX_NAME, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -269,10 +317,12 @@ void query_some_results_with_query() { @Test void query_some_results_with_scroll() { mockTwoPageResponse(client); - final var requestuilder = new OpenSearchRequestBuilder(10, exprValueFactory); + final var requestuilder = new OpenSearchRequestBuilder(10, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, 3, requestuilder.build(INDEX_NAME, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE))) { + client, + 3, + requestuilder.build(INDEX_NAME, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -306,12 +356,13 @@ void query_results_limited_by_query_size() { }); final int defaultQuerySize = 2; - final var requestBuilder = new OpenSearchRequestBuilder(defaultQuerySize, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(defaultQuerySize, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( client, defaultQuerySize, - requestBuilder.build(INDEX_NAME, QUERY_SIZE, CURSOR_KEEP_ALIVE))) { + requestBuilder.build(INDEX_NAME, QUERY_SIZE, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -368,7 +419,7 @@ void push_down_highlight_with_arguments() { } private PushDownAssertion assertThat() { - return new PushDownAssertion(client, exprValueFactory); + return new PushDownAssertion(client, exprValueFactory, settings); } private static class PushDownAssertion { @@ -377,9 +428,10 @@ private static class PushDownAssertion { private final OpenSearchResponse response; private final OpenSearchExprValueFactory factory; - public PushDownAssertion(OpenSearchClient client, OpenSearchExprValueFactory valueFactory) { + public PushDownAssertion( + OpenSearchClient client, OpenSearchExprValueFactory valueFactory, Settings settings) { this.client = client; - this.requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, valueFactory); + this.requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, valueFactory, settings); this.response = mock(OpenSearchResponse.class); this.factory = valueFactory; @@ -411,7 +463,9 @@ PushDownAssertion shouldQueryHighlight(QueryBuilder query, HighlightBuilder high when(client.search(request)).thenReturn(response); var indexScan = new OpenSearchIndexScan( - client, QUERY_SIZE, requestBuilder.build(EMPLOYEES_INDEX, 10000, CURSOR_KEEP_ALIVE)); + client, + QUERY_SIZE, + requestBuilder.build(EMPLOYEES_INDEX, 10000, CURSOR_KEEP_ALIVE, client)); indexScan.open(); return this; } @@ -429,7 +483,9 @@ PushDownAssertion shouldQuery(QueryBuilder expected) { when(client.search(request)).thenReturn(response); var indexScan = new OpenSearchIndexScan( - client, 10000, requestBuilder.build(EMPLOYEES_INDEX, 10000, CURSOR_KEEP_ALIVE)); + client, + 10000, + requestBuilder.build(EMPLOYEES_INDEX, 10000, CURSOR_KEEP_ALIVE, client)); indexScan.open(); return this; } From 173737c744877fc624cb603e6b7018f50dd11d73 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 24 Oct 2024 18:56:50 -0700 Subject: [PATCH 25/32] [Manual Backport] Backport doctest fix to 2.x (#3087) Signed-off-by: Simeon Widdis --- docs/user/beyond/partiql.rst | 20 +- docs/user/dql/aggregations.rst | 216 +- docs/user/dql/basics.rst | 98 +- docs/user/dql/complex.rst | 28 +- docs/user/dql/expressions.rst | 149 +- docs/user/dql/functions.rst | 1824 ++++++++--------- docs/user/dql/metadata.rst | 76 +- docs/user/dql/window.rst | 208 +- docs/user/general/comments.rst | 20 +- docs/user/general/datatypes.rst | 60 +- docs/user/general/identifiers.rst | 76 +- docs/user/general/values.rst | 32 +- .../admin/connectors/prometheus_connector.rst | 80 +- docs/user/ppl/admin/cross_cluster_search.rst | 16 +- docs/user/ppl/cmd/ad.rst | 44 +- docs/user/ppl/cmd/dedup.rst | 70 +- docs/user/ppl/cmd/describe.rst | 50 +- docs/user/ppl/cmd/eval.rst | 48 +- docs/user/ppl/cmd/fields.rst | 32 +- docs/user/ppl/cmd/grok.rst | 16 +- docs/user/ppl/cmd/head.rst | 44 +- docs/user/ppl/cmd/information_schema.rst | 30 +- docs/user/ppl/cmd/ml.rst | 44 +- docs/user/ppl/cmd/parse.rst | 14 +- docs/user/ppl/cmd/patterns.rst | 16 +- docs/user/ppl/cmd/rare.rst | 28 +- docs/user/ppl/cmd/rename.rst | 32 +- docs/user/ppl/cmd/search.rst | 28 +- docs/user/ppl/cmd/showdatasources.rst | 10 +- docs/user/ppl/cmd/sort.rst | 80 +- docs/user/ppl/cmd/stats.rst | 250 +-- docs/user/ppl/cmd/top.rst | 34 +- docs/user/ppl/cmd/where.rst | 12 +- docs/user/ppl/functions/condition.rst | 132 +- docs/user/ppl/functions/conversion.rst | 30 +- docs/user/ppl/functions/datetime.rst | 860 ++++---- docs/user/ppl/functions/expressions.rst | 60 +- docs/user/ppl/functions/math.rst | 212 +- docs/user/ppl/functions/relevance.rst | 110 +- docs/user/ppl/functions/string.rst | 130 +- docs/user/ppl/functions/system.rst | 10 +- docs/user/ppl/general/datatypes.rst | 30 +- docs/user/ppl/general/identifiers.rst | 62 +- doctest/build.gradle | 12 +- doctest/test_docs.py | 28 +- 45 files changed, 2759 insertions(+), 2702 deletions(-) diff --git a/docs/user/beyond/partiql.rst b/docs/user/beyond/partiql.rst index 76fec8405d..6a93a55c94 100644 --- a/docs/user/beyond/partiql.rst +++ b/docs/user/beyond/partiql.rst @@ -202,11 +202,11 @@ Selecting top level for object fields, object fields of array value and nested f os> SELECT city, accounts, projects FROM people; fetched rows / total rows = 1/1 - +-----------------------------------------------------+------------+----------------------------------------------------------------------------------------------------------------+ - | city | accounts | projects | - |-----------------------------------------------------+------------+----------------------------------------------------------------------------------------------------------------| - | {'name': 'Seattle', 'location': {'latitude': 10.5}} | {'id': 1} | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] | - +-----------------------------------------------------+------------+----------------------------------------------------------------------------------------------------------------+ + +-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------+ + | city | accounts | projects | + |-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------| + | {'name': 'Seattle', 'location': {'latitude': 10.5}} | {'id': 1} | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] | + +-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------+ Example 2: Selecting Deeper Levels ---------------------------------- @@ -215,11 +215,11 @@ Selecting at deeper levels for object fields of regular value returns inner fiel os> SELECT city.location, city.location.latitude FROM people; fetched rows / total rows = 1/1 - +--------------------+--------------------------+ - | city.location | city.location.latitude | - |--------------------+--------------------------| - | {'latitude': 10.5} | 10.5 | - +--------------------+--------------------------+ + +--------------------+------------------------+ + | city.location | city.location.latitude | + |--------------------+------------------------| + | {'latitude': 10.5} | 10.5 | + +--------------------+------------------------+ For selecting second level for nested fields, please read on and find more details in the following sections. diff --git a/docs/user/dql/aggregations.rst b/docs/user/dql/aggregations.rst index 42db4cdb4f..adf933c09c 100644 --- a/docs/user/dql/aggregations.rst +++ b/docs/user/dql/aggregations.rst @@ -34,12 +34,12 @@ The group by expression could be identifier:: os> SELECT gender, sum(age) FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | sum(age) | - |----------+------------| - | F | 28 | - | M | 101 | - +----------+------------+ + +--------+----------+ + | gender | sum(age) | + |--------+----------| + | F | 28 | + | M | 101 | + +--------+----------+ Ordinal @@ -49,12 +49,12 @@ The group by expression could be ordinal:: os> SELECT gender, sum(age) FROM accounts GROUP BY 1; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | sum(age) | - |----------+------------| - | F | 28 | - | M | 101 | - +----------+------------+ + +--------+----------+ + | gender | sum(age) | + |--------+----------| + | F | 28 | + | M | 101 | + +--------+----------+ Expression @@ -64,14 +64,14 @@ The group by expression could be expression:: os> SELECT abs(account_number), sum(age) FROM accounts GROUP BY abs(account_number); fetched rows / total rows = 4/4 - +-----------------------+------------+ - | abs(account_number) | sum(age) | - |-----------------------+------------| - | 1 | 32 | - | 13 | 28 | - | 18 | 33 | - | 6 | 36 | - +-----------------------+------------+ + +---------------------+----------+ + | abs(account_number) | sum(age) | + |---------------------+----------| + | 1 | 32 | + | 13 | 28 | + | 18 | 33 | + | 6 | 36 | + +---------------------+----------+ Aggregation @@ -91,12 +91,12 @@ The aggregation could be used select:: os> SELECT gender, sum(age) FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | sum(age) | - |----------+------------| - | F | 28 | - | M | 101 | - +----------+------------+ + +--------+----------+ + | gender | sum(age) | + |--------+----------| + | F | 28 | + | M | 101 | + +--------+----------+ Expression over Aggregation --------------------------- @@ -105,12 +105,12 @@ The aggregation could be used as arguments of expression:: os> SELECT gender, sum(age) * 2 as sum2 FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+--------+ - | gender | sum2 | - |----------+--------| - | F | 56 | - | M | 202 | - +----------+--------+ + +--------+------+ + | gender | sum2 | + |--------+------| + | F | 56 | + | M | 202 | + +--------+------+ Expression as argument of Aggregation ------------------------------------- @@ -119,12 +119,12 @@ The aggregation could has expression as arguments:: os> SELECT gender, sum(age * 2) as sum2 FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+--------+ - | gender | sum2 | - |----------+--------| - | F | 56 | - | M | 202 | - +----------+--------+ + +--------+------+ + | gender | sum2 | + |--------+------| + | F | 56 | + | M | 202 | + +--------+------+ COUNT Aggregations ------------------ @@ -150,12 +150,12 @@ Example:: os> SELECT gender, count(*) as countV FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+----------+ - | gender | countV | - |----------+----------| - | F | 1 | - | M | 3 | - +----------+----------+ + +--------+--------+ + | gender | countV | + |--------+--------| + | F | 1 | + | M | 3 | + +--------+--------+ SUM --- @@ -169,12 +169,12 @@ Example:: os> SELECT gender, sum(age) as sumV FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+--------+ - | gender | sumV | - |----------+--------| - | F | 28 | - | M | 101 | - +----------+--------+ + +--------+------+ + | gender | sumV | + |--------+------| + | F | 28 | + | M | 101 | + +--------+------+ AVG --- @@ -188,12 +188,12 @@ Example:: os> SELECT gender, avg(age) as avgV FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+--------------------+ - | gender | avgV | - |----------+--------------------| - | F | 28.0 | - | M | 33.666666666666664 | - +----------+--------------------+ + +--------+--------------------+ + | gender | avgV | + |--------+--------------------| + | F | 28.0 | + | M | 33.666666666666664 | + +--------+--------------------+ MAX --- @@ -207,11 +207,11 @@ Example:: os> SELECT max(age) as maxV FROM accounts; fetched rows / total rows = 1/1 - +--------+ - | maxV | - |--------| - | 36 | - +--------+ + +------+ + | maxV | + |------| + | 36 | + +------+ MIN --- @@ -225,11 +225,11 @@ Example:: os> SELECT min(age) as minV FROM accounts; fetched rows / total rows = 1/1 - +--------+ - | minV | - |--------| - | 28 | - +--------+ + +------+ + | minV | + |------| + | 28 | + +------+ VAR_POP ------- @@ -364,11 +364,11 @@ To get the count of distinct values of a field, you can add a keyword ``DISTINCT os> SELECT COUNT(DISTINCT gender), COUNT(gender) FROM accounts; fetched rows / total rows = 1/1 - +--------------------------+-----------------+ - | COUNT(DISTINCT gender) | COUNT(gender) | - |--------------------------+-----------------| - | 2 | 4 | - +--------------------------+-----------------+ + +------------------------+---------------+ + | COUNT(DISTINCT gender) | COUNT(gender) | + |------------------------+---------------| + | 2 | 4 | + +------------------------+---------------+ PERCENTILE or PERCENTILE_APPROX ------------------------------- @@ -382,12 +382,12 @@ Example:: os> SELECT gender, percentile(age, 90) as p90 FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+-------+ - | gender | p90 | - |----------+-------| - | F | 28 | - | M | 36 | - +----------+-------+ + +--------+-----+ + | gender | p90 | + |--------+-----| + | F | 28 | + | M | 36 | + +--------+-----+ HAVING Clause ============= @@ -413,11 +413,11 @@ Here is an example for typical use of ``HAVING`` clause:: ... GROUP BY gender ... HAVING sum(age) > 100; fetched rows / total rows = 1/1 - +----------+------------+ - | gender | sum(age) | - |----------+------------| - | M | 101 | - +----------+------------+ + +--------+----------+ + | gender | sum(age) | + |--------+----------| + | M | 101 | + +--------+----------+ Here is another example for using alias in ``HAVING`` condition. Note that if an identifier is ambiguous, for example present both as a select alias and an index field, preference is alias. This means the identifier will be replaced by expression aliased in ``SELECT`` clause:: @@ -427,11 +427,11 @@ Here is another example for using alias in ``HAVING`` condition. Note that if an ... GROUP BY gender ... HAVING s > 100; fetched rows / total rows = 1/1 - +----------+-----+ - | gender | s | - |----------+-----| - | M | 101 | - +----------+-----+ + +--------+-----+ + | gender | s | + |--------+-----| + | M | 101 | + +--------+-----+ HAVING without GROUP BY ----------------------- @@ -443,11 +443,11 @@ Additionally, a ``HAVING`` clause can work without ``GROUP BY`` clause. This is ... FROM accounts ... HAVING sum(age) > 100; fetched rows / total rows = 1/1 - +------------------------+ - | 'Total of age > 100' | - |------------------------| - | Total of age > 100 | - +------------------------+ + +----------------------+ + | 'Total of age > 100' | + |----------------------| + | Total of age > 100 | + +----------------------+ FILTER Clause @@ -465,12 +465,12 @@ The group by aggregation with ``FILTER`` clause can set different conditions for os> SELECT avg(age) FILTER(WHERE balance > 10000) AS filtered, gender FROM accounts GROUP BY gender fetched rows / total rows = 2/2 - +------------+----------+ - | filtered | gender | - |------------+----------| - | 28.0 | F | - | 32.0 | M | - +------------+----------+ + +----------+--------+ + | filtered | gender | + |----------+--------| + | 28.0 | F | + | 32.0 | M | + +----------+--------+ FILTER without GROUP BY ----------------------- @@ -482,11 +482,11 @@ The ``FILTER`` clause can be used in aggregation functions without GROUP BY as w ... count(*) FILTER(WHERE age > 34) AS filtered ... FROM accounts fetched rows / total rows = 1/1 - +--------------+------------+ - | unfiltered | filtered | - |--------------+------------| - | 4 | 1 | - +--------------+------------+ + +------------+----------+ + | unfiltered | filtered | + |------------+----------| + | 4 | 1 | + +------------+----------+ Distinct count aggregate with FILTER ------------------------------------ @@ -495,9 +495,9 @@ The ``FILTER`` clause is also used in distinct count to do the filtering before os> SELECT COUNT(DISTINCT firstname) FILTER(WHERE age > 30) AS distinct_count FROM accounts fetched rows / total rows = 1/1 - +------------------+ - | distinct_count | - |------------------| - | 3 | - +------------------+ + +----------------+ + | distinct_count | + |----------------| + | 3 | + +----------------+ diff --git a/docs/user/dql/basics.rst b/docs/user/dql/basics.rst index a03ac4db70..a59f193086 100644 --- a/docs/user/dql/basics.rst +++ b/docs/user/dql/basics.rst @@ -191,14 +191,14 @@ This produces results like this for example:: os> SELECT firstname, lastname, _index, _sort FROM accounts; fetched rows / total rows = 4/4 - +-------------+------------+----------+---------+ - | firstname | lastname | _index | _sort | - |-------------+------------+----------+---------| - | Amber | Duke | accounts | -2 | - | Hattie | Bond | accounts | -2 | - | Nanette | Bates | accounts | -2 | - | Dale | Adams | accounts | -2 | - +-------------+------------+----------+---------+ + +-----------+----------+----------+-------+ + | firstname | lastname | _index | _sort | + |-----------+----------+----------+-------| + | Amber | Duke | accounts | -2 | + | Hattie | Bond | accounts | -2 | + | Nanette | Bates | accounts | -2 | + | Dale | Adams | accounts | -2 | + +-----------+----------+----------+-------+ Example 3: Using Field Alias ---------------------------- @@ -303,13 +303,13 @@ In fact your can use any expression in a ``DISTINCT`` clause as follows:: os> SELECT DISTINCT SUBSTRING(lastname, 1, 1) FROM accounts; fetched rows / total rows = 3/3 - +-----------------------------+ - | SUBSTRING(lastname, 1, 1) | - |-----------------------------| - | A | - | B | - | D | - +-----------------------------+ + +---------------------------+ + | SUBSTRING(lastname, 1, 1) | + |---------------------------| + | A | + | B | + | D | + +---------------------------+ FROM ==== @@ -988,14 +988,14 @@ Note that the example above is essentially sorting on a predicate expression. In os> SELECT employer FROM accounts ORDER BY employer ASC NULLS LAST; fetched rows / total rows = 4/4 - +------------+ - | employer | - |------------| - | Netagy | - | Pyrami | - | Quility | - | null | - +------------+ + +----------+ + | employer | + |----------| + | Netagy | + | Pyrami | + | Quility | + | null | + +----------+ The sorting rule can be summarized as follows: @@ -1010,14 +1010,14 @@ Here is another example for sort in descending order without ``NULLS`` clause:: os> SELECT employer FROM accounts ORDER BY employer DESC; fetched rows / total rows = 4/4 - +------------+ - | employer | - |------------| - | Quility | - | Pyrami | - | Netagy | - | null | - +------------+ + +----------+ + | employer | + |----------| + | Quility | + | Pyrami | + | Netagy | + | null | + +----------+ Example 3: Ordering by Aggregate Functions @@ -1027,23 +1027,23 @@ Aggregate functions are allowed to be used in ``ORDER BY`` clause. You can refer os> SELECT gender, MAX(age) FROM accounts GROUP BY gender ORDER BY MAX(age) DESC; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | MAX(age) | - |----------+------------| - | M | 36 | - | F | 28 | - +----------+------------+ + +--------+----------+ + | gender | MAX(age) | + |--------+----------| + | M | 36 | + | F | 28 | + +--------+----------+ Even if it's not present in ``SELECT`` clause, it can be also used as follows:: os> SELECT gender, MIN(age) FROM accounts GROUP BY gender ORDER BY MAX(age) DESC; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | MIN(age) | - |----------+------------| - | M | 32 | - | F | 28 | - +----------+------------+ + +--------+----------+ + | gender | MIN(age) | + |--------+----------| + | M | 32 | + | F | 28 | + +--------+----------+ LIMIT ===== @@ -1147,12 +1147,12 @@ Offset position can be given following the OFFSET keyword as well, here is an ex >od SELECT age FROM accounts ORDER BY age LIMIT 2 OFFSET 1 fetched rows / total rows = 2/2 - +-------+ - | age | - |-------| - | 32 | - | 33 | - +-------+ + +-----+ + | age | + |-----| + | 32 | + | 33 | + +-----+ Limitation diff --git a/docs/user/dql/complex.rst b/docs/user/dql/complex.rst index 17009d712b..906ea21904 100644 --- a/docs/user/dql/complex.rst +++ b/docs/user/dql/complex.rst @@ -247,14 +247,14 @@ Here is another example with aggregation function and GROUP BY in subquery:: ... SELECT AVG(balance) AS avg_balance FROM accounts GROUP BY gender, age ... ) AS a; fetched rows / total rows = 4/4 - +---------------+ - | avg_balance | - |---------------| - | 32838.0 | - | 39225.0 | - | 4180.0 | - | 5686.0 | - +---------------+ + +-------------+ + | avg_balance | + |-------------| + | 32838.0 | + | 39225.0 | + | 4180.0 | + | 5686.0 | + +-------------+ Query with multiple layers of subquery is supported as well, here follows a example:: @@ -265,12 +265,12 @@ Query with multiple layers of subquery is supported as well, here follows a exam ... ) AS accounts WHERE age < 35 ... ) AS accounts fetched rows / total rows = 2/2 - +--------+ - | name | - |--------| - | Duke | - | Adams | - +--------+ + +-------+ + | name | + |-------| + | Duke | + | Adams | + +-------+ JOINs diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 94cfcba68d..4816121be2 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -34,20 +34,29 @@ Here is an example for different type of literals:: os> SELECT 123, 'hello', false, -4.567, DATE '2020-07-07', TIME '01:01:01', TIMESTAMP '2020-07-07 01:01:01'; fetched rows / total rows = 1/1 - +-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------+ - | 123 | 'hello' | false | -4.567 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | - |-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------| - | 123 | hello | False | -4.567 | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | - +-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------+ + +-----+---------+-------+--------+-------------------+-----------------+---------------------------------+ + | 123 | 'hello' | false | -4.567 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | + |-----+---------+-------+--------+-------------------+-----------------+---------------------------------| + | 123 | hello | False | -4.567 | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | + +-----+---------+-------+--------+-------------------+-----------------+---------------------------------+ + + + os> SELECT "Hello", 'Hello', "It""s", 'It''s', "It's", '"Its"', 'It\'s', 'It\\\'s', "\I\t\s" + fetched rows / total rows = 1/1 + +---------+---------+---------+---------+--------+---------+---------+-----------+----------+ + | "Hello" | 'Hello' | "It""s" | 'It''s' | "It's" | '"Its"' | 'It\'s' | 'It\\\'s' | "\I\t\s" | + |---------+---------+---------+---------+--------+---------+---------+-----------+----------| + | Hello | Hello | It"s | It's | It's | "Its" | It\'s | It\\\'s | \I\t\s | + +---------+---------+---------+---------+--------+---------+---------+-----------+----------+ os> SELECT {DATE '2020-07-07'}, {D '2020-07-07'}, {TIME '01:01:01'}, {T '01:01:01'}, {TIMESTAMP '2020-07-07 01:01:01'}, {TS '2020-07-07 01:01:01'} fetched rows / total rows = 1/1 - +-----------------------+--------------------+---------------------+------------------+-------------------------------------+------------------------------+ - | {DATE '2020-07-07'} | {D '2020-07-07'} | {TIME '01:01:01'} | {T '01:01:01'} | {TIMESTAMP '2020-07-07 01:01:01'} | {TS '2020-07-07 01:01:01'} | - |-----------------------+--------------------+---------------------+------------------+-------------------------------------+------------------------------| - | 2020-07-07 | 2020-07-07 | 01:01:01 | 01:01:01 | 2020-07-07 01:01:01 | 2020-07-07 01:01:01 | - +-----------------------+--------------------+---------------------+------------------+-------------------------------------+------------------------------+ + +---------------------+------------------+-------------------+----------------+-----------------------------------+----------------------------+ + | {DATE '2020-07-07'} | {D '2020-07-07'} | {TIME '01:01:01'} | {T '01:01:01'} | {TIMESTAMP '2020-07-07 01:01:01'} | {TS '2020-07-07 01:01:01'} | + |---------------------+------------------+-------------------+----------------+-----------------------------------+----------------------------| + | 2020-07-07 | 2020-07-07 | 01:01:01 | 01:01:01 | 2020-07-07 01:01:01 | 2020-07-07 01:01:01 | + +---------------------+------------------+-------------------+----------------+-----------------------------------+----------------------------+ Limitations ----------- @@ -93,11 +102,11 @@ Here is an example for different type of arithmetic expressions:: os> SELECT 1 + 2, (9 - 1) % 3, 2 * 4 / 3; fetched rows / total rows = 1/1 - +---------+---------------+-------------+ - | 1 + 2 | (9 - 1) % 3 | 2 * 4 / 3 | - |---------+---------------+-------------| - | 3 | 2 | 2 | - +---------+---------------+-------------+ + +-------+-------------+-----------+ + | 1 + 2 | (9 - 1) % 3 | 2 * 4 / 3 | + |-------+-------------+-----------| + | 3 | 2 | 2 | + +-------+-------------+-----------+ Comparison Operators ================================== @@ -153,38 +162,38 @@ Here is an example for different type of comparison operators:: os> SELECT 2 > 1, 2 >= 1, 2 < 1, 2 != 1, 2 <= 1, 2 = 1; fetched rows / total rows = 1/1 - +---------+----------+---------+----------+----------+---------+ - | 2 > 1 | 2 >= 1 | 2 < 1 | 2 != 1 | 2 <= 1 | 2 = 1 | - |---------+----------+---------+----------+----------+---------| - | True | True | False | True | False | False | - +---------+----------+---------+----------+----------+---------+ + +-------+--------+-------+--------+--------+-------+ + | 2 > 1 | 2 >= 1 | 2 < 1 | 2 != 1 | 2 <= 1 | 2 = 1 | + |-------+--------+-------+--------+--------+-------| + | True | True | False | True | False | False | + +-------+--------+-------+--------+--------+-------+ It is possible to compare datetimes. When comparing different datetime types, for example `DATE` and `TIME`, both converted to `DATETIME`. The following rule is applied on coversion: a `TIME` applied to today's date; `DATE` is interpreted at midnight. See example below:: os> SELECT current_time() > current_date() AS `now.time > today`, typeof(current_time()) AS `now.time.type`, typeof(current_date()) AS `now.date.type`; fetched rows / total rows = 1/1 - +--------------------+-----------------+-----------------+ - | now.time > today | now.time.type | now.date.type | - |--------------------+-----------------+-----------------| - | True | TIME | DATE | - +--------------------+-----------------+-----------------+ + +------------------+---------------+---------------+ + | now.time > today | now.time.type | now.date.type | + |------------------+---------------+---------------| + | True | TIME | DATE | + +------------------+---------------+---------------+ os> SELECT current_time() = now() AS `now.time = now`, typeof(current_time()) AS `now.time.type`, typeof(now()) AS `now.type`; fetched rows / total rows = 1/1 - +------------------+-----------------+------------+ - | now.time = now | now.time.type | now.type | - |------------------+-----------------+------------| - | True | TIME | DATETIME | - +------------------+-----------------+------------+ + +----------------+---------------+----------+ + | now.time = now | now.time.type | now.type | + |----------------+---------------+----------| + | True | TIME | DATETIME | + +----------------+---------------+----------+ os> SELECT subtime(now(), current_time()) = current_date() AS `midnight = now.date`, typeof(subtime(now(), current_time())) AS `midnight.type`, typeof(current_date()) AS `now.date.type`; fetched rows / total rows = 1/1 - +-----------------------+-----------------+-----------------+ - | midnight = now.date | midnight.type | now.date.type | - |-----------------------+-----------------+-----------------| - | True | DATETIME | DATE | - +-----------------------+-----------------+-----------------+ + +---------------------+---------------+---------------+ + | midnight = now.date | midnight.type | now.date.type | + |---------------------+---------------+---------------| + | True | DATETIME | DATE | + +---------------------+---------------+---------------+ LIKE @@ -194,11 +203,11 @@ expr LIKE pattern. The expr is string value, pattern is supports literal text, a os> SELECT 'axyzb' LIKE 'a%b', 'acb' LIKE 'A_B', 'axyzb' NOT LIKE 'a%b', 'acb' NOT LIKE 'a_b'; fetched rows / total rows = 1/1 - +----------------------+--------------------+--------------------------+------------------------+ - | 'axyzb' LIKE 'a%b' | 'acb' LIKE 'A_B' | 'axyzb' NOT LIKE 'a%b' | 'acb' NOT LIKE 'a_b' | - |----------------------+--------------------+--------------------------+------------------------| - | True | True | False | False | - +----------------------+--------------------+--------------------------+------------------------+ + +--------------------+------------------+------------------------+----------------------+ + | 'axyzb' LIKE 'a%b' | 'acb' LIKE 'A_B' | 'axyzb' NOT LIKE 'a%b' | 'acb' NOT LIKE 'a_b' | + |--------------------+------------------+------------------------+----------------------| + | True | True | False | False | + +--------------------+------------------+------------------------+----------------------+ NULL value test --------------- @@ -207,11 +216,11 @@ Here is an example for null value test:: os> SELECT 0 IS NULL, 0 IS NOT NULL, NULL IS NULL, NULL IS NOT NULL; fetched rows / total rows = 1/1 - +-------------+-----------------+----------------+--------------------+ - | 0 IS NULL | 0 IS NOT NULL | NULL IS NULL | NULL IS NOT NULL | - |-------------+-----------------+----------------+--------------------| - | False | True | True | False | - +-------------+-----------------+----------------+--------------------+ + +-----------+---------------+--------------+------------------+ + | 0 IS NULL | 0 IS NOT NULL | NULL IS NULL | NULL IS NOT NULL | + |-----------+---------------+--------------+------------------| + | False | True | True | False | + +-----------+---------------+--------------+------------------+ REGEXP value test @@ -221,11 +230,11 @@ expr REGEXP pattern. The expr is string value, pattern is supports regular expre os> SELECT 'Hello!' REGEXP '.*', 'a' REGEXP 'b'; fetched rows / total rows = 1/1 - +------------------------+------------------+ - | 'Hello!' REGEXP '.*' | 'a' REGEXP 'b' | - |------------------------+------------------| - | 1 | 0 | - +------------------------+------------------+ + +----------------------+----------------+ + | 'Hello!' REGEXP '.*' | 'a' REGEXP 'b' | + |----------------------+----------------| + | 1 | 0 | + +----------------------+----------------+ IN value list test ------------------ @@ -234,11 +243,11 @@ Here is an example for IN value test:: os> SELECT 1 in (1, 2), 3 not in (1, 2); fetched rows / total rows = 1/1 - +---------------+-------------------+ - | 1 in (1, 2) | 3 not in (1, 2) | - |---------------+-------------------| - | True | True | - +---------------+-------------------+ + +-------------+-----------------+ + | 1 in (1, 2) | 3 not in (1, 2) | + |-------------+-----------------| + | True | True | + +-------------+-----------------+ BETWEEN range test ------------------ @@ -250,11 +259,11 @@ Here is an example for range test by BETWEEN expression:: ... 4 BETWEEN 1 AND 3, ... 4 NOT BETWEEN 1 AND 3; fetched rows / total rows = 1/1 - +---------------------+---------------------+-------------------------+ - | 1 BETWEEN 1 AND 3 | 4 BETWEEN 1 AND 3 | 4 NOT BETWEEN 1 AND 3 | - |---------------------+---------------------+-------------------------| - | True | False | True | - +---------------------+---------------------+-------------------------+ + +-------------------+-------------------+-----------------------+ + | 1 BETWEEN 1 AND 3 | 4 BETWEEN 1 AND 3 | 4 NOT BETWEEN 1 AND 3 | + |-------------------+-------------------+-----------------------| + | True | False | True | + +-------------------+-------------------+-----------------------+ Function Call @@ -284,11 +293,11 @@ Here is an example for different type of arithmetic expressions:: os> SELECT abs(-1.234), abs(-1 * abs(-5)); fetched rows / total rows = 1/1 - +---------------+---------------------+ - | abs(-1.234) | abs(-1 * abs(-5)) | - |---------------+---------------------| - | 1.234 | 5 | - +---------------+---------------------+ + +-------------+-------------------+ + | abs(-1.234) | abs(-1 * abs(-5)) | + |-------------+-------------------| + | 1.234 | 5 | + +-------------+-------------------+ Date function examples ---------------------- @@ -297,11 +306,11 @@ Here is an example for different type of arithmetic expressions:: os> SELECT dayofmonth(DATE '2020-07-07'); fetched rows / total rows = 1/1 - +---------------------------------+ - | dayofmonth(DATE '2020-07-07') | - |---------------------------------| - | 7 | - +---------------------------------+ + +-------------------------------+ + | dayofmonth(DATE '2020-07-07') | + |-------------------------------| + | 7 | + +-------------------------------+ Limitations ----------- diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index d76f2e3442..ac782a1881 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -49,21 +49,21 @@ Cast to string example:: os> SELECT cast(true as string) as cbool, cast(1 as string) as cint, cast(DATE '2012-08-07' as string) as cdate fetched rows / total rows = 1/1 - +---------+--------+------------+ - | cbool | cint | cdate | - |---------+--------+------------| - | true | 1 | 2012-08-07 | - +---------+--------+------------+ + +-------+------+------------+ + | cbool | cint | cdate | + |-------+------+------------| + | true | 1 | 2012-08-07 | + +-------+------+------------+ Cast to number example:: os> SELECT cast(true as int) as cbool, cast('1' as integer) as cstring fetched rows / total rows = 1/1 - +---------+-----------+ - | cbool | cstring | - |---------+-----------| - | 1 | 1 | - +---------+-----------+ + +-------+---------+ + | cbool | cstring | + |-------+---------| + | 1 | 1 | + +-------+---------+ Cast to date example:: @@ -79,11 +79,11 @@ Cast function can be chained:: os> SELECT cast(cast(true as string) as boolean) as cbool fetched rows / total rows = 1/1 - +---------+ - | cbool | - |---------| - | True | - +---------+ + +-------+ + | cbool | + |-------| + | True | + +-------+ Mathematical Functions @@ -103,11 +103,11 @@ Example:: os> SELECT ABS(0), ABS(10), ABS(-10), ABS(12.34567), ABS(-12.34567) fetched rows / total rows = 1/1 - +----------+-----------+------------+-----------------+------------------+ - | ABS(0) | ABS(10) | ABS(-10) | ABS(12.34567) | ABS(-12.34567) | - |----------+-----------+------------+-----------------+------------------| - | 0 | 10 | 10 | 12.34567 | 12.34567 | - +----------+-----------+------------+-----------------+------------------+ + +--------+---------+----------+---------------+----------------+ + | ABS(0) | ABS(10) | ABS(-10) | ABS(12.34567) | ABS(-12.34567) | + |--------+---------+----------+---------------+----------------| + | 0 | 10 | 10 | 12.34567 | 12.34567 | + +--------+---------+----------+---------------+----------------+ ACOS @@ -151,11 +151,11 @@ Example:: os> SELECT ADD(2, 1), ADD(2.5, 3); fetched rows / total rows = 1/1 - +-------------+---------------+ - | ADD(2, 1) | ADD(2.5, 3) | - |-------------+---------------| - | 3 | 5.5 | - +-------------+---------------+ + +-----------+-------------+ + | ADD(2, 1) | ADD(2.5, 3) | + |-----------+-------------| + | 3 | 5.5 | + +-----------+-------------+ ASIN ---- @@ -173,11 +173,11 @@ Example:: os> SELECT ASIN(0) fetched rows / total rows = 1/1 - +-----------+ - | ASIN(0) | - |-----------| - | 0.0 | - +-----------+ + +---------+ + | ASIN(0) | + |---------| + | 0.0 | + +---------+ ATAN @@ -244,11 +244,11 @@ Example:: os> SELECT CBRT(8), CBRT(9.261), CBRT(-27); fetched rows / total rows = 1/1 - +-----------+---------------+-------------+ - | CBRT(8) | CBRT(9.261) | CBRT(-27) | - |-----------+---------------+-------------| - | 2.0 | 2.1 | -3.0 | - +-----------+---------------+-------------+ + +---------+-------------+-----------+ + | CBRT(8) | CBRT(9.261) | CBRT(-27) | + |---------+-------------+-----------| + | 2.0 | 2.1 | -3.0 | + +---------+-------------+-----------+ CEIL @@ -281,29 +281,29 @@ Example:: os> SELECT CEILING(0), CEILING(50.00005), CEILING(-50.00005); fetched rows / total rows = 1/1 - +--------------+---------------------+----------------------+ - | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | - |--------------+---------------------+----------------------| - | 0 | 51 | -50 | - +--------------+---------------------+----------------------+ + +------------+-------------------+--------------------+ + | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | + |------------+-------------------+--------------------| + | 0 | 51 | -50 | + +------------+-------------------+--------------------+ os> SELECT CEILING(3147483647.12345), CEILING(113147483647.12345), CEILING(3147483647.00001); fetched rows / total rows = 1/1 - +-----------------------------+-------------------------------+-----------------------------+ - | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | - |-----------------------------+-------------------------------+-----------------------------| - | 3147483648 | 113147483648 | 3147483648 | - +-----------------------------+-------------------------------+-----------------------------+ + +---------------------------+-----------------------------+---------------------------+ + | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | + |---------------------------+-----------------------------+---------------------------| + | 3147483648 | 113147483648 | 3147483648 | + +---------------------------+-----------------------------+---------------------------+ Example:: os> SELECT CEIL(0), CEIL(12.34567), CEIL(-12.34567) fetched rows / total rows = 1/1 - +-----------+------------------+-------------------+ - | CEIL(0) | CEIL(12.34567) | CEIL(-12.34567) | - |-----------+------------------+-------------------| - | 0 | 13 | -12 | - +-----------+------------------+-------------------+ + +---------+----------------+-----------------+ + | CEIL(0) | CEIL(12.34567) | CEIL(-12.34567) | + |---------+----------------+-----------------| + | 0 | 13 | -12 | + +---------+----------------+-----------------+ CONV @@ -322,11 +322,11 @@ Example:: os> SELECT CONV('12', 10, 16), CONV('2C', 16, 10), CONV(12, 10, 2), CONV(1111, 2, 10) fetched rows / total rows = 1/1 - +----------------------+----------------------+-------------------+---------------------+ - | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | - |----------------------+----------------------+-------------------+---------------------| - | c | 44 | 1100 | 15 | - +----------------------+----------------------+-------------------+---------------------+ + +--------------------+--------------------+-----------------+-------------------+ + | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | + |--------------------+--------------------+-----------------+-------------------| + | c | 44 | 1100 | 15 | + +--------------------+--------------------+-----------------+-------------------+ COS @@ -345,11 +345,11 @@ Example:: os> SELECT COS(0) fetched rows / total rows = 1/1 - +----------+ - | COS(0) | - |----------| - | 1.0 | - +----------+ + +--------+ + | COS(0) | + |--------| + | 1.0 | + +--------+ COSH @@ -414,11 +414,11 @@ Example:: os> SELECT CRC32('MySQL') fetched rows / total rows = 1/1 - +------------------+ - | CRC32('MySQL') | - |------------------| - | 3259397556 | - +------------------+ + +----------------+ + | CRC32('MySQL') | + |----------------| + | 3259397556 | + +----------------+ DEGREES @@ -462,11 +462,11 @@ Example:: os> SELECT DIVIDE(10, 2), DIVIDE(7.5, 3); fetched rows / total rows = 1/1 - +-----------------+------------------+ - | DIVIDE(10, 2) | DIVIDE(7.5, 3) | - |-----------------+------------------| - | 5 | 2.5 | - +-----------------+------------------+ + +---------------+----------------+ + | DIVIDE(10, 2) | DIVIDE(7.5, 3) | + |---------------+----------------| + | 5 | 2.5 | + +---------------+----------------+ E @@ -533,11 +533,11 @@ Example:: os> SELECT EXPM1(-1), EXPM1(0), EXPM1(1), EXPM1(1.5) fetched rows / total rows = 1/1 - +---------------------+------------+-------------------+-------------------+ - | EXPM1(-1) | EXPM1(0) | EXPM1(1) | EXPM1(1.5) | - |---------------------+------------+-------------------+-------------------| - | -0.6321205588285577 | 0.0 | 1.718281828459045 | 3.481689070338065 | - +---------------------+------------+-------------------+-------------------+ + +---------------------+----------+-------------------+-------------------+ + | EXPM1(-1) | EXPM1(0) | EXPM1(1) | EXPM1(1.5) | + |---------------------+----------+-------------------+-------------------| + | -0.6321205588285577 | 0.0 | 1.718281828459045 | 3.481689070338065 | + +---------------------+----------+-------------------+-------------------+ FLOOR @@ -558,27 +558,27 @@ Example:: os> SELECT FLOOR(0), FLOOR(50.00005), FLOOR(-50.00005); fetched rows / total rows = 1/1 - +------------+-------------------+--------------------+ - | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | - |------------+-------------------+--------------------| - | 0 | 50 | -51 | - +------------+-------------------+--------------------+ + +----------+-----------------+------------------+ + | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | + |----------+-----------------+------------------| + | 0 | 50 | -51 | + +----------+-----------------+------------------+ os> SELECT FLOOR(3147483647.12345), FLOOR(113147483647.12345), FLOOR(3147483647.00001); fetched rows / total rows = 1/1 - +---------------------------+-----------------------------+---------------------------+ - | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | - |---------------------------+-----------------------------+---------------------------| - | 3147483647 | 113147483647 | 3147483647 | - +---------------------------+-----------------------------+---------------------------+ + +-------------------------+---------------------------+-------------------------+ + | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | + |-------------------------+---------------------------+-------------------------| + | 3147483647 | 113147483647 | 3147483647 | + +-------------------------+---------------------------+-------------------------+ os> SELECT FLOOR(282474973688888.022), FLOOR(9223372036854775807.022), FLOOR(9223372036854775807.0000001); fetched rows / total rows = 1/1 - +------------------------------+----------------------------------+--------------------------------------+ - | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | - |------------------------------+----------------------------------+--------------------------------------| - | 282474973688888 | 9223372036854775807 | 9223372036854775807 | - +------------------------------+----------------------------------+--------------------------------------+ + +----------------------------+--------------------------------+------------------------------------+ + | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | + |----------------------------+--------------------------------+------------------------------------| + | 282474973688888 | 9223372036854775807 | 9223372036854775807 | + +----------------------------+--------------------------------+------------------------------------+ LN @@ -597,11 +597,11 @@ Example:: os> select LN(1), LN(e()), LN(10), LN(12.34567); fetched rows / total rows = 1/1 - +---------+-----------+-------------------+--------------------+ - | LN(1) | LN(e()) | LN(10) | LN(12.34567) | - |---------+-----------+-------------------+--------------------| - | 0.0 | 1.0 | 2.302585092994046 | 2.5133053943094317 | - +---------+-----------+-------------------+--------------------+ + +-------+---------+-------------------+--------------------+ + | LN(1) | LN(e()) | LN(10) | LN(12.34567) | + |-------+---------+-------------------+--------------------| + | 0.0 | 1.0 | 2.302585092994046 | 2.5133053943094317 | + +-------+---------+-------------------+--------------------+ LOG @@ -623,11 +623,11 @@ Example:: os> select LOG(1), LOG(e()), LOG(2, 65536), LOG(10, 10000); fetched rows / total rows = 1/1 - +----------+------------+-----------------+------------------+ - | LOG(1) | LOG(e()) | LOG(2, 65536) | LOG(10, 10000) | - |----------+------------+-----------------+------------------| - | 0.0 | 1.0 | 16.0 | 4.0 | - +----------+------------+-----------------+------------------+ + +--------+----------+---------------+----------------+ + | LOG(1) | LOG(e()) | LOG(2, 65536) | LOG(10, 10000) | + |--------+----------+---------------+----------------| + | 0.0 | 1.0 | 16.0 | 4.0 | + +--------+----------+---------------+----------------+ LOG2 @@ -646,11 +646,11 @@ Example:: os> select LOG2(1), LOG2(8), LOG2(65536), LOG2(8.8245); fetched rows / total rows = 1/1 - +-----------+-----------+---------------+--------------------+ - | LOG2(1) | LOG2(8) | LOG2(65536) | LOG2(8.8245) | - |-----------+-----------+---------------+--------------------| - | 0.0 | 3.0 | 16.0 | 3.1415145369723745 | - +-----------+-----------+---------------+--------------------+ + +---------+---------+-------------+--------------------+ + | LOG2(1) | LOG2(8) | LOG2(65536) | LOG2(8.8245) | + |---------+---------+-------------+--------------------| + | 0.0 | 3.0 | 16.0 | 3.1415145369723745 | + +---------+---------+-------------+--------------------+ LOG10 @@ -669,11 +669,11 @@ Example:: os> select LOG10(1), LOG10(8), LOG10(1000), LOG10(8.8245); fetched rows / total rows = 1/1 - +------------+--------------------+---------------+--------------------+ - | LOG10(1) | LOG10(8) | LOG10(1000) | LOG10(8.8245) | - |------------+--------------------+---------------+--------------------| - | 0.0 | 0.9030899869919435 | 3.0 | 0.9456901074431278 | - +------------+--------------------+---------------+--------------------+ + +----------+--------------------+-------------+--------------------+ + | LOG10(1) | LOG10(8) | LOG10(1000) | LOG10(8.8245) | + |----------+--------------------+-------------+--------------------| + | 0.0 | 0.9030899869919435 | 3.0 | 0.9456901074431278 | + +----------+--------------------+-------------+--------------------+ MOD @@ -694,11 +694,11 @@ Example:: os> SELECT MOD(3, 2), MOD(3.1, 2) fetched rows / total rows = 1/1 - +-------------+---------------+ - | MOD(3, 2) | MOD(3.1, 2) | - |-------------+---------------| - | 1 | 1.1 | - +-------------+---------------+ + +-----------+-------------+ + | MOD(3, 2) | MOD(3.1, 2) | + |-----------+-------------| + | 1 | 1.1 | + +-----------+-------------+ MODULUS ------- @@ -718,11 +718,11 @@ Example:: os> SELECT MODULUS(3, 2), MODULUS(3.1, 2) fetched rows / total rows = 1/1 - +-----------------+-------------------+ - | MODULUS(3, 2) | MODULUS(3.1, 2) | - |-----------------+-------------------| - | 1 | 1.1 | - +-----------------+-------------------+ + +---------------+-----------------+ + | MODULUS(3, 2) | MODULUS(3.1, 2) | + |---------------+-----------------| + | 1 | 1.1 | + +---------------+-----------------+ MULTIPLY @@ -743,11 +743,11 @@ Example:: os> SELECT MULTIPLY(1, 2), MULTIPLY(-2, 1), MULTIPLY(1.5, 2); fetched rows / total rows = 1/1 - +------------------+-------------------+--------------------+ - | MULTIPLY(1, 2) | MULTIPLY(-2, 1) | MULTIPLY(1.5, 2) | - |------------------+-------------------+--------------------| - | 2 | -2 | 3.0 | - +------------------+-------------------+--------------------+ + +----------------+-----------------+------------------+ + | MULTIPLY(1, 2) | MULTIPLY(-2, 1) | MULTIPLY(1.5, 2) | + |----------------+-----------------+------------------| + | 2 | -2 | 3.0 | + +----------------+-----------------+------------------+ PI @@ -789,11 +789,11 @@ Example:: os> SELECT POW(3, 2), POW(-3, 2), POW(3, -2) fetched rows / total rows = 1/1 - +-------------+--------------+--------------------+ - | POW(3, 2) | POW(-3, 2) | POW(3, -2) | - |-------------+--------------+--------------------| - | 9.0 | 9.0 | 0.1111111111111111 | - +-------------+--------------+--------------------+ + +-----------+------------+--------------------+ + | POW(3, 2) | POW(-3, 2) | POW(3, -2) | + |-----------+------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +-----------+------------+--------------------+ POWER @@ -814,11 +814,11 @@ Example:: os> SELECT POWER(3, 2), POWER(-3, 2), POWER(3, -2) fetched rows / total rows = 1/1 - +---------------+----------------+--------------------+ - | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | - |---------------+----------------+--------------------| - | 9.0 | 9.0 | 0.1111111111111111 | - +---------------+----------------+--------------------+ + +-------------+--------------+--------------------+ + | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | + |-------------+--------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +-------------+--------------+--------------------+ RADIANS @@ -883,11 +883,11 @@ Example:: os> SELECT RINT(1.7); fetched rows / total rows = 1/1 - +-------------+ - | RINT(1.7) | - |-------------| - | 2.0 | - +-------------+ + +-----------+ + | RINT(1.7) | + |-----------| + | 2.0 | + +-----------+ ROUND @@ -910,11 +910,11 @@ Example:: os> SELECT ROUND(12.34), ROUND(12.34, 1), ROUND(12.34, -1), ROUND(12, 1) fetched rows / total rows = 1/1 - +----------------+-------------------+--------------------+----------------+ - | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | - |----------------+-------------------+--------------------+----------------| - | 12.0 | 12.3 | 10.0 | 12 | - +----------------+-------------------+--------------------+----------------+ + +--------------+-----------------+------------------+--------------+ + | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | + |--------------+-----------------+------------------+--------------| + | 12.0 | 12.3 | 10.0 | 12 | + +--------------+-----------------+------------------+--------------+ SIGN @@ -933,11 +933,11 @@ Example:: os> SELECT SIGN(1), SIGN(0), SIGN(-1.1) fetched rows / total rows = 1/1 - +-----------+-----------+--------------+ - | SIGN(1) | SIGN(0) | SIGN(-1.1) | - |-----------+-----------+--------------| - | 1 | 0 | -1 | - +-----------+-----------+--------------+ + +---------+---------+------------+ + | SIGN(1) | SIGN(0) | SIGN(-1.1) | + |---------+---------+------------| + | 1 | 0 | -1 | + +---------+---------+------------+ SIGNUM @@ -958,11 +958,11 @@ Example:: os> SELECT SIGNUM(1), SIGNUM(0), SIGNUM(-1.1) fetched rows / total rows = 1/1 - +-------------+-------------+----------------+ - | SIGNUM(1) | SIGNUM(0) | SIGNUM(-1.1) | - |-------------+-------------+----------------| - | 1 | 0 | -1 | - +-------------+-------------+----------------+ + +-----------+-----------+--------------+ + | SIGNUM(1) | SIGNUM(0) | SIGNUM(-1.1) | + |-----------+-----------+--------------| + | 1 | 0 | -1 | + +-----------+-----------+--------------+ SIN @@ -981,11 +981,11 @@ Example:: os> select sin(0), sin(1), sin(pi()), abs(sin(pi())) < 0.0001; fetched rows / total rows = 1/1 - +----------+--------------------+------------------------+---------------------------+ - | sin(0) | sin(1) | sin(pi()) | abs(sin(pi())) < 0.0001 | - |----------+--------------------+------------------------+---------------------------| - | 0.0 | 0.8414709848078965 | 1.2246467991473532e-16 | True | - +----------+--------------------+------------------------+---------------------------+ + +--------+--------------------+------------------------+-------------------------+ + | sin(0) | sin(1) | sin(pi()) | abs(sin(pi())) < 0.0001 | + |--------+--------------------+------------------------+-------------------------| + | 0.0 | 0.8414709848078965 | 1.2246467991473532e-16 | True | + +--------+--------------------+------------------------+-------------------------+ SINH @@ -1030,11 +1030,11 @@ Example:: os> SELECT SQRT(4), SQRT(4.41) fetched rows / total rows = 1/1 - +-----------+--------------+ - | SQRT(4) | SQRT(4.41) | - |-----------+--------------| - | 2.0 | 2.1 | - +-----------+--------------+ + +---------+------------+ + | SQRT(4) | SQRT(4.41) | + |---------+------------| + | 2.0 | 2.1 | + +---------+------------+ STRCMP @@ -1053,11 +1053,11 @@ Example:: os> SELECT STRCMP('hello', 'world'), STRCMP('hello', 'hello') fetched rows / total rows = 1/1 - +----------------------------+----------------------------+ - | STRCMP('hello', 'world') | STRCMP('hello', 'hello') | - |----------------------------+----------------------------| - | -1 | 0 | - +----------------------------+----------------------------+ + +--------------------------+--------------------------+ + | STRCMP('hello', 'world') | STRCMP('hello', 'hello') | + |--------------------------+--------------------------| + | -1 | 0 | + +--------------------------+--------------------------+ SUBTRACT @@ -1078,11 +1078,11 @@ Example:: os> SELECT SUBTRACT(2, 1), SUBTRACT(2.5, 3); fetched rows / total rows = 1/1 - +------------------+--------------------+ - | SUBTRACT(2, 1) | SUBTRACT(2.5, 3) | - |------------------+--------------------| - | 1 | -0.5 | - +------------------+--------------------+ + +----------------+------------------+ + | SUBTRACT(2, 1) | SUBTRACT(2.5, 3) | + |----------------+------------------| + | 1 | -0.5 | + +----------------+------------------+ TAN @@ -1101,11 +1101,11 @@ Example:: os> SELECT TAN(0) fetched rows / total rows = 1/1 - +----------+ - | TAN(0) | - |----------| - | 0.0 | - +----------+ + +--------+ + | TAN(0) | + |--------| + | 0.0 | + +--------+ TRUNCATE @@ -1126,11 +1126,11 @@ FLOAT/DOUBLE -> DOUBLE Example:: fetched rows / total rows = 1/1 - +----------------------+-----------------------+-------------------+ - | TRUNCATE(56.78, 1) | TRUNCATE(56.78, -1) | TRUNCATE(56, 1) | - |----------------------+-----------------------+-------------------| - | 56.7 | 50 | 56 | - +----------------------+-----------------------+-------------------+ + +--------------------+---------------------+-----------------+ + | TRUNCATE(56.78, 1) | TRUNCATE(56.78, -1) | TRUNCATE(56, 1) | + |--------------------+---------------------+-----------------| + | 56.7 | 50 | 56 | + +--------------------+---------------------+-----------------+ Date and Time Functions @@ -1163,11 +1163,11 @@ Example:: os> SELECT ADDDATE(DATE('2020-08-26'), INTERVAL 1 HOUR) AS `'2020-08-26' + 1h`, ADDDATE(DATE('2020-08-26'), 1) AS `'2020-08-26' + 1`, ADDDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) AS `ts '2020-08-26 01:01:01' + 1` fetched rows / total rows = 1/1 - +---------------------+--------------------+--------------------------------+ - | '2020-08-26' + 1h | '2020-08-26' + 1 | ts '2020-08-26 01:01:01' + 1 | - |---------------------+--------------------+--------------------------------| - | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | - +---------------------+--------------------+--------------------------------+ + +---------------------+------------------+------------------------------+ + | '2020-08-26' + 1h | '2020-08-26' + 1 | ts '2020-08-26 01:01:01' + 1 | + |---------------------+------------------+------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | + +---------------------+------------------+------------------------------+ ADDTIME @@ -1200,35 +1200,35 @@ Example:: os> SELECT ADDTIME(TIME('23:59:59'), DATE('2004-01-01')) AS `'23:59:59' + 0` fetched rows / total rows = 1/1 - +------------------+ - | '23:59:59' + 0 | - |------------------| - | 23:59:59 | - +------------------+ + +----------------+ + | '23:59:59' + 0 | + |----------------| + | 23:59:59 | + +----------------+ os> SELECT ADDTIME(DATE('2004-01-01'), TIME('23:59:59')) AS `'2004-01-01' + '23:59:59'` fetched rows / total rows = 1/1 - +-----------------------------+ - | '2004-01-01' + '23:59:59' | - |-----------------------------| - | 2004-01-01 23:59:59 | - +-----------------------------+ - - os> SELECT ADDTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' + '00:05:42'` - fetched rows / total rows = 1/1 +---------------------------+ - | '10:20:30' + '00:05:42' | + | '2004-01-01' + '23:59:59' | |---------------------------| - | 10:26:12 | + | 2004-01-01 23:59:59 | +---------------------------+ + os> SELECT ADDTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' + '00:05:42'` + fetched rows / total rows = 1/1 + +-------------------------+ + | '10:20:30' + '00:05:42' | + |-------------------------| + | 10:26:12 | + +-------------------------+ + os> SELECT ADDTIME(TIMESTAMP('2007-02-28 10:20:30'), DATETIME('2002-03-04 20:40:50')) AS `'2007-02-28 10:20:30' + '20:40:50'` fetched rows / total rows = 1/1 - +--------------------------------------+ - | '2007-02-28 10:20:30' + '20:40:50' | - |--------------------------------------| - | 2007-03-01 07:01:20 | - +--------------------------------------+ + +------------------------------------+ + | '2007-02-28 10:20:30' + '20:40:50' | + |------------------------------------| + | 2007-03-01 07:01:20 | + +------------------------------------+ CONVERT_TZ @@ -1247,86 +1247,86 @@ Example:: os> SELECT CONVERT_TZ('2008-12-25 05:30:00', '+00:00', 'America/Los_Angeles') fetched rows / total rows = 1/1 - +----------------------------------------------------------------------+ - | CONVERT_TZ('2008-12-25 05:30:00', '+00:00', 'America/Los_Angeles') | - |----------------------------------------------------------------------| - | 2008-12-24 21:30:00 | - +----------------------------------------------------------------------+ + +--------------------------------------------------------------------+ + | CONVERT_TZ('2008-12-25 05:30:00', '+00:00', 'America/Los_Angeles') | + |--------------------------------------------------------------------| + | 2008-12-24 21:30:00 | + +--------------------------------------------------------------------+ os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-10:00") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-10:00") | - |---------------------------------------------------------| - | 2010-10-09 23:10:10 | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-10:00") | + |-------------------------------------------------------| + | 2010-10-09 23:10:10 | + +-------------------------------------------------------+ When the datedate, or either of the two time zone fields are invalid format, then the result is null. In this example any datetime that is not will result in null. Example:: os> SELECT CONVERT_TZ("test", "+01:00", "-10:00") fetched rows / total rows = 1/1 - +------------------------------------------+ - | CONVERT_TZ("test", "+01:00", "-10:00") | - |------------------------------------------| - | null | - +------------------------------------------+ + +----------------------------------------+ + | CONVERT_TZ("test", "+01:00", "-10:00") | + |----------------------------------------| + | null | + +----------------------------------------+ When the datetime, or either of the two time zone fields are invalid format, then the result is null. In this example any timezone that is not <+HH:mm> or <-HH:mm> will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "test", "-10:00") fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "test", "-10:00") | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "test", "-10:00") | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:00") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:00") | - |---------------------------------------------------------| - | 2010-10-10 23:10:10 | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:00") | + |-------------------------------------------------------| + | 2010-10-10 23:10:10 | + +-------------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:01") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:01") | - |---------------------------------------------------------| - | null | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:01") | + |-------------------------------------------------------| + | null | + +-------------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-13:59") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-13:59") | - |---------------------------------------------------------| - | 2010-10-09 19:11:10 | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-13:59") | + |-------------------------------------------------------| + | 2010-10-09 19:11:10 | + +-------------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-14:00") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-14:00") | - |---------------------------------------------------------| - | null | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-14:00") | + |-------------------------------------------------------| + | null | + +-------------------------------------------------------+ CURDATE @@ -1346,11 +1346,11 @@ Example:: > SELECT CURDATE(); fetched rows / total rows = 1/1 - +-------------+ - | CURDATE() | - |-------------| - | 2022-08-02 | - +-------------+ + +------------+ + | CURDATE() | + |------------| + | 2022-08-02 | + +------------+ CURRENT_DATE @@ -1365,11 +1365,11 @@ Example:: > SELECT CURRENT_DATE(); fetched rows / total rows = 1/1 - +------------------+ - | CURRENT_DATE() | - |------------------+ - | 2022-08-02 | - +------------------+ + +----------------+ + | CURRENT_DATE() | + |----------------+ + | 2022-08-02 | + +----------------+ CURRENT_TIME @@ -1384,11 +1384,11 @@ Example:: > SELECT CURRENT_TIME(); fetched rows / total rows = 1/1 - +-----------------+ - | CURRENT_TIME() | - |-----------------+ - | 15:39:05 | - +-----------------+ + +----------------+ + | CURRENT_TIME() | + |----------------+ + | 15:39:05 | + +----------------+ CURRENT_TIMESTAMP @@ -1403,11 +1403,11 @@ Example:: > SELECT CURRENT_TIMESTAMP(); fetched rows / total rows = 1/1 - +-----------------------+ - | CURRENT_TIMESTAMP() | - |-----------------------+ - | 2022-08-02 15:54:19 | - +-----------------------+ + +---------------------+ + | CURRENT_TIMESTAMP() | + |---------------------+ + | 2022-08-02 15:54:19 | + +---------------------+ CURTIME @@ -1427,11 +1427,11 @@ Example:: > SELECT CURTIME() as value_1, CURTIME() as value_2; fetched rows / total rows = 1/1 - +-----------+-----------+ - | value_1 | value_2 | - |-----------+-----------| - | 15:39:05 | 15:39:05 | - +-----------+-----------+ + +----------+----------+ + | value_1 | value_2 | + |----------+----------| + | 15:39:05 | 15:39:05 | + +----------+----------+ DATE @@ -1450,11 +1450,11 @@ Example:: os> SELECT DATE('2020-08-26'), DATE(TIMESTAMP('2020-08-26 13:49:00')), DATE('2020-08-26 13:49:00'), DATE('2020-08-26 13:49') fetched rows / total rows = 1/1 - +----------------------+------------------------------------------+-------------------------------+----------------------------+ - | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | DATE('2020-08-26 13:49:00') | DATE('2020-08-26 13:49') | - |----------------------+------------------------------------------+-------------------------------+----------------------------| - | 2020-08-26 | 2020-08-26 | 2020-08-26 | 2020-08-26 | - +----------------------+------------------------------------------+-------------------------------+----------------------------+ + +--------------------+----------------------------------------+-----------------------------+--------------------------+ + | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | DATE('2020-08-26 13:49:00') | DATE('2020-08-26 13:49') | + |--------------------+----------------------------------------+-----------------------------+--------------------------| + | 2020-08-26 | 2020-08-26 | 2020-08-26 | 2020-08-26 | + +--------------------+----------------------------------------+-----------------------------+--------------------------+ DATETIME @@ -1477,44 +1477,44 @@ Example:: os> SELECT DATETIME('2008-12-25 05:30:00+00:00', 'America/Los_Angeles') fetched rows / total rows = 1/1 - +----------------------------------------------------------------+ - | DATETIME('2008-12-25 05:30:00+00:00', 'America/Los_Angeles') | - |----------------------------------------------------------------| - | 2008-12-24 21:30:00 | - +----------------------------------------------------------------+ + +--------------------------------------------------------------+ + | DATETIME('2008-12-25 05:30:00+00:00', 'America/Los_Angeles') | + |--------------------------------------------------------------| + | 2008-12-24 21:30:00 | + +--------------------------------------------------------------+ This example converts from -10:00 timezone to +10:00 timezone. Example:: os> SELECT DATETIME('2004-02-28 23:00:00-10:00', '+10:00') fetched rows / total rows = 1/1 - +---------------------------------------------------+ - | DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | - |---------------------------------------------------| - | 2004-02-29 19:00:00 | - +---------------------------------------------------+ + +-------------------------------------------------+ + | DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | + |-------------------------------------------------| + | 2004-02-29 19:00:00 | + +-------------------------------------------------+ This example uses the timezone -14:00, which is outside of the range -13:59 and +14:00. This results in a null value. Example:: os> SELECT DATETIME('2008-01-01 02:00:00', '-14:00') fetched rows / total rows = 1/1 - +---------------------------------------------+ - | DATETIME('2008-01-01 02:00:00', '-14:00') | - |---------------------------------------------| - | null | - +---------------------------------------------+ + +-------------------------------------------+ + | DATETIME('2008-01-01 02:00:00', '-14:00') | + |-------------------------------------------| + | null | + +-------------------------------------------+ February 30th is not a day, so it returns null. Example:: os> SELECT DATETIME('2008-02-30 02:00:00', '-00:00') fetched rows / total rows = 1/1 - +---------------------------------------------+ - | DATETIME('2008-02-30 02:00:00', '-00:00') | - |---------------------------------------------| - | null | - +---------------------------------------------+ + +-------------------------------------------+ + | DATETIME('2008-02-30 02:00:00', '-00:00') | + |-------------------------------------------| + | null | + +-------------------------------------------+ DATETIME(datetime) examples @@ -1523,33 +1523,33 @@ Example:: os> SELECT DATETIME('2008-02-10 02:00:00') fetched rows / total rows = 1/1 - +-----------------------------------+ - | DATETIME('2008-02-10 02:00:00') | - |-----------------------------------| - | 2008-02-10 02:00:00 | - +-----------------------------------+ + +---------------------------------+ + | DATETIME('2008-02-10 02:00:00') | + |---------------------------------| + | 2008-02-10 02:00:00 | + +---------------------------------+ February 30th is not a day, so it returns null. Example:: os> SELECT DATETIME('2008-02-30 02:00:00') fetched rows / total rows = 1/1 - +-----------------------------------+ - | DATETIME('2008-02-30 02:00:00') | - |-----------------------------------| - | null | - +-----------------------------------+ + +---------------------------------+ + | DATETIME('2008-02-30 02:00:00') | + |---------------------------------| + | null | + +---------------------------------+ DATETIME with a datetime and no seperate timezone to convert to returns the datetime object without a timezone. Example:: os> SELECT DATETIME('2008-02-10 02:00:00+04:00') fetched rows / total rows = 1/1 - +-----------------------------------------+ - | DATETIME('2008-02-10 02:00:00+04:00') | - |-----------------------------------------| - | 2008-02-10 02:00:00 | - +-----------------------------------------+ + +---------------------------------------+ + | DATETIME('2008-02-10 02:00:00+04:00') | + |---------------------------------------| + | 2008-02-10 02:00:00 | + +---------------------------------------+ DATE_ADD @@ -1572,11 +1572,11 @@ Example:: os> SELECT DATE_ADD(DATE('2020-08-26'), INTERVAL 1 HOUR) AS `'2020-08-26' + 1h`, DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 1 DAY) as `ts '2020-08-26 01:01:01' + 1d` fetched rows / total rows = 1/1 - +---------------------+---------------------------------+ - | '2020-08-26' + 1h | ts '2020-08-26 01:01:01' + 1d | - |---------------------+---------------------------------| - | 2020-08-26 01:00:00 | 2020-08-27 01:01:01 | - +---------------------+---------------------------------+ + +---------------------+-------------------------------+ + | '2020-08-26' + 1h | ts '2020-08-26 01:01:01' + 1d | + |---------------------+-------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 01:01:01 | + +---------------------+-------------------------------+ DATE_FORMAT @@ -1671,11 +1671,11 @@ Example:: os> SELECT DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f'), DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') fetched rows / total rows = 1/1 - +------------------------------------------------------+-----------------------------------------------------------------------+ - | DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f') | DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | - |------------------------------------------------------+-----------------------------------------------------------------------| - | 13:14:15.012345 | 1998-Jan-31st 01:14:15 PM | - +------------------------------------------------------+-----------------------------------------------------------------------+ + +----------------------------------------------------+---------------------------------------------------------------------+ + | DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f') | DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | + |----------------------------------------------------+---------------------------------------------------------------------| + | 13:14:15.012345 | 1998-Jan-31st 01:14:15 PM | + +----------------------------------------------------+---------------------------------------------------------------------+ DATE_SUB @@ -1698,11 +1698,11 @@ Example:: os> SELECT DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY) AS `'2008-01-02' - 31d`, DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 1 HOUR) AS `ts '2020-08-26 01:01:01' + 1h` fetched rows / total rows = 1/1 - +----------------------+---------------------------------+ - | '2008-01-02' - 31d | ts '2020-08-26 01:01:01' + 1h | - |----------------------+---------------------------------| - | 2007-12-02 00:00:00 | 2020-08-26 00:01:01 | - +----------------------+---------------------------------+ + +---------------------+-------------------------------+ + | '2008-01-02' - 31d | ts '2020-08-26 01:01:01' + 1h | + |---------------------+-------------------------------| + | 2007-12-02 00:00:00 | 2020-08-26 00:01:01 | + +---------------------+-------------------------------+ DATEDIFF @@ -1718,11 +1718,11 @@ Example:: os> SELECT DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59')) AS `'2000-01-02' - '2000-01-01'`, DATEDIFF(DATE('2001-02-01'), TIMESTAMP('2004-01-01 00:00:00')) AS `'2001-02-01' - '2004-01-01'`, DATEDIFF(TIME('23:59:59'), TIME('00:00:00')) AS `today - today` fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+-----------------+ - | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | - |-------------------------------+-------------------------------+-----------------| - | 1 | -1064 | 0 | - +-------------------------------+-------------------------------+-----------------+ + +-----------------------------+-----------------------------+---------------+ + | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | + |-----------------------------+-----------------------------+---------------| + | 1 | -1064 | 0 | + +-----------------------------+-----------------------------+---------------+ DAY @@ -1743,11 +1743,11 @@ Example:: os> SELECT DAY(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +---------------------------+ - | DAY(DATE('2020-08-26')) | - |---------------------------| - | 26 | - +---------------------------+ + +-------------------------+ + | DAY(DATE('2020-08-26')) | + |-------------------------| + | 26 | + +-------------------------+ DAYNAME @@ -1766,11 +1766,11 @@ Example:: os> SELECT DAYNAME(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-------------------------------+ - | DAYNAME(DATE('2020-08-26')) | - |-------------------------------| - | Wednesday | - +-------------------------------+ + +-----------------------------+ + | DAYNAME(DATE('2020-08-26')) | + |-----------------------------| + | Wednesday | + +-----------------------------+ DAYOFMONTH @@ -1791,11 +1791,11 @@ Example:: os> SELECT DAYOFMONTH(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +----------------------------------+ - | DAYOFMONTH(DATE('2020-08-26')) | - |----------------------------------| - | 26 | - +----------------------------------+ + +--------------------------------+ + | DAYOFMONTH(DATE('2020-08-26')) | + |--------------------------------| + | 26 | + +--------------------------------+ DAY_OF_MONTH @@ -1816,11 +1816,11 @@ Example:: os> SELECT DAY_OF_MONTH('2020-08-26') fetched rows / total rows = 1/1 - +------------------------------+ - | DAY_OF_MONTH('2020-08-26') | - |------------------------------| - | 26 | - +------------------------------+ + +----------------------------+ + | DAY_OF_MONTH('2020-08-26') | + |----------------------------| + | 26 | + +----------------------------+ DAYOFWEEK @@ -1841,11 +1841,11 @@ Example:: os> SELECT DAYOFWEEK('2020-08-26'), DAY_OF_WEEK('2020-08-26') fetched rows / total rows = 1/1 - +---------------------------+-----------------------------+ - | DAYOFWEEK('2020-08-26') | DAY_OF_WEEK('2020-08-26') | - |---------------------------+-----------------------------| - | 4 | 4 | - +---------------------------+-----------------------------+ + +-------------------------+---------------------------+ + | DAYOFWEEK('2020-08-26') | DAY_OF_WEEK('2020-08-26') | + |-------------------------+---------------------------| + | 4 | 4 | + +-------------------------+---------------------------+ DAYOFYEAR @@ -1866,27 +1866,27 @@ Example:: os> SELECT DAYOFYEAR(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +---------------------------------+ - | DAYOFYEAR(DATE('2020-08-26')) | - |---------------------------------| - | 239 | - +---------------------------------+ + +-------------------------------+ + | DAYOFYEAR(DATE('2020-08-26')) | + |-------------------------------| + | 239 | + +-------------------------------+ os> SELECT DAYOFYEAR(DATETIME('2020-08-26 00:00:00')) fetched rows / total rows = 1/1 - +----------------------------------------------+ - | DAYOFYEAR(DATETIME('2020-08-26 00:00:00')) | - |----------------------------------------------| - | 239 | - +----------------------------------------------+ + +--------------------------------------------+ + | DAYOFYEAR(DATETIME('2020-08-26 00:00:00')) | + |--------------------------------------------| + | 239 | + +--------------------------------------------+ os> SELECT DAYOFYEAR(TIMESTAMP('2020-08-26 00:00:00')) fetched rows / total rows = 1/1 - +-----------------------------------------------+ - | DAYOFYEAR(TIMESTAMP('2020-08-26 00:00:00')) | - |-----------------------------------------------| - | 239 | - +-----------------------------------------------+ + +---------------------------------------------+ + | DAYOFYEAR(TIMESTAMP('2020-08-26 00:00:00')) | + |---------------------------------------------| + | 239 | + +---------------------------------------------+ DAY_OF_YEAR @@ -1906,27 +1906,27 @@ Example:: os> SELECT DAY_OF_YEAR(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-----------------------------------+ - | DAY_OF_YEAR(DATE('2020-08-26')) | - |-----------------------------------| - | 239 | - +-----------------------------------+ + +---------------------------------+ + | DAY_OF_YEAR(DATE('2020-08-26')) | + |---------------------------------| + | 239 | + +---------------------------------+ os> SELECT DAY_OF_YEAR(DATETIME('2020-08-26 00:00:00')) fetched rows / total rows = 1/1 - +------------------------------------------------+ - | DAY_OF_YEAR(DATETIME('2020-08-26 00:00:00')) | - |------------------------------------------------| - | 239 | - +------------------------------------------------+ + +----------------------------------------------+ + | DAY_OF_YEAR(DATETIME('2020-08-26 00:00:00')) | + |----------------------------------------------| + | 239 | + +----------------------------------------------+ os> SELECT DAY_OF_YEAR(TIMESTAMP('2020-08-26 00:00:00')) fetched rows / total rows = 1/1 - +-------------------------------------------------+ - | DAY_OF_YEAR(TIMESTAMP('2020-08-26 00:00:00')) | - |-------------------------------------------------| - | 239 | - +-------------------------------------------------+ + +-----------------------------------------------+ + | DAY_OF_YEAR(TIMESTAMP('2020-08-26 00:00:00')) | + |-----------------------------------------------| + | 239 | + +-----------------------------------------------+ EXTRACT @@ -1994,11 +1994,11 @@ Example:: os> SELECT extract(YEAR_MONTH FROM "2023-02-07 10:11:12"); fetched rows / total rows = 1/1 - +--------------------------------------------------+ - | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | - |--------------------------------------------------| - | 202302 | - +--------------------------------------------------+ + +------------------------------------------------+ + | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | + |------------------------------------------------| + | 202302 | + +------------------------------------------------+ FROM_DAYS @@ -2017,11 +2017,11 @@ Example:: os> SELECT FROM_DAYS(733687) fetched rows / total rows = 1/1 - +---------------------+ - | FROM_DAYS(733687) | - |---------------------| - | 2008-10-07 | - +---------------------+ + +-------------------+ + | FROM_DAYS(733687) | + |-------------------| + | 2008-10-07 | + +-------------------+ FROM_UNIXTIME @@ -2046,19 +2046,19 @@ Examples:: os> select FROM_UNIXTIME(1220249547) fetched rows / total rows = 1/1 - +-----------------------------+ - | FROM_UNIXTIME(1220249547) | - |-----------------------------| - | 2008-09-01 06:12:27 | - +-----------------------------+ + +---------------------------+ + | FROM_UNIXTIME(1220249547) | + |---------------------------| + | 2008-09-01 06:12:27 | + +---------------------------+ os> select FROM_UNIXTIME(1220249547, '%T') fetched rows / total rows = 1/1 - +-----------------------------------+ - | FROM_UNIXTIME(1220249547, '%T') | - |-----------------------------------| - | 06:12:27 | - +-----------------------------------+ + +---------------------------------+ + | FROM_UNIXTIME(1220249547, '%T') | + |---------------------------------| + | 06:12:27 | + +---------------------------------+ GET_FORMAT @@ -2077,11 +2077,11 @@ Examples:: os> select GET_FORMAT(DATE, 'USA'); fetched rows / total rows = 1/1 - +---------------------------+ - | GET_FORMAT(DATE, 'USA') | - |---------------------------| - | %m.%d.%Y | - +---------------------------+ + +-------------------------+ + | GET_FORMAT(DATE, 'USA') | + |-------------------------| + | %m.%d.%Y | + +-------------------------+ HOUR @@ -2101,11 +2101,11 @@ Example:: os> SELECT HOUR('01:02:03'), HOUR_OF_DAY('01:02:03') fetched rows / total rows = 1/1 - +--------------------+---------------------------+ - | HOUR('01:02:03') | HOUR_OF_DAY('01:02:03') | - |--------------------+---------------------------| - | 1 | 1 | - +--------------------+---------------------------+ + +------------------+-------------------------+ + | HOUR('01:02:03') | HOUR_OF_DAY('01:02:03') | + |------------------+-------------------------| + | 1 | 1 | + +------------------+-------------------------+ LAST_DAY @@ -2121,11 +2121,11 @@ Example:: os> SELECT last_day('2023-02-06'); fetched rows / total rows = 1/1 - +--------------------------+ - | last_day('2023-02-06') | - |--------------------------| - | 2023-02-28 | - +--------------------------+ + +------------------------+ + | last_day('2023-02-06') | + |------------------------| + | 2023-02-28 | + +------------------------+ LOCALTIMESTAMP @@ -2193,11 +2193,11 @@ Example:: os> select MAKEDATE(1945, 5.9), MAKEDATE(1984, 1984) fetched rows / total rows = 1/1 - +-----------------------+------------------------+ - | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | - |-----------------------+------------------------| - | 1945-01-06 | 1989-06-06 | - +-----------------------+------------------------+ + +---------------------+----------------------+ + | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | + |---------------------+----------------------| + | 1945-01-06 | 1989-06-06 | + +---------------------+----------------------+ MAKETIME @@ -2225,11 +2225,11 @@ Example:: os> select MAKETIME(20, 30, 40), MAKETIME(20.2, 49.5, 42.100502) fetched rows / total rows = 1/1 - +------------------------+-----------------------------------+ - | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | - |------------------------+-----------------------------------| - | 20:30:40 | 20:50:42.100502 | - +------------------------+-----------------------------------+ + +----------------------+---------------------------------+ + | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | + |----------------------+---------------------------------| + | 20:30:40 | 20:50:42.100502 | + +----------------------+---------------------------------+ MICROSECOND @@ -2248,11 +2248,11 @@ Example:: os> SELECT MICROSECOND((TIME '01:02:03.123456')) fetched rows / total rows = 1/1 - +-----------------------------------------+ - | MICROSECOND((TIME '01:02:03.123456')) | - |-----------------------------------------| - | 123456 | - +-----------------------------------------+ + +---------------------------------------+ + | MICROSECOND((TIME '01:02:03.123456')) | + |---------------------------------------| + | 123456 | + +---------------------------------------+ MINUTE @@ -2272,11 +2272,11 @@ Example:: os> SELECT MINUTE(time('01:02:03')), MINUTE_OF_HOUR(time('01:02:03')) fetched rows / total rows = 1/1 - +----------------------------+------------------------------------+ - | MINUTE(time('01:02:03')) | MINUTE_OF_HOUR(time('01:02:03')) | - |----------------------------+------------------------------------| - | 2 | 2 | - +----------------------------+------------------------------------+ + +--------------------------+----------------------------------+ + | MINUTE(time('01:02:03')) | MINUTE_OF_HOUR(time('01:02:03')) | + |--------------------------+----------------------------------| + | 2 | 2 | + +--------------------------+----------------------------------+ MINUTE_OF_DAY @@ -2295,11 +2295,11 @@ Example:: os> SELECT MINUTE_OF_DAY((TIME '01:02:03')) fetched rows / total rows = 1/1 - +------------------------------------+ - | MINUTE_OF_DAY((TIME '01:02:03')) | - |------------------------------------| - | 62 | - +------------------------------------+ + +----------------------------------+ + | MINUTE_OF_DAY((TIME '01:02:03')) | + |----------------------------------| + | 62 | + +----------------------------------+ MONTH @@ -2320,20 +2320,20 @@ Example:: os> SELECT MONTH(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-----------------------------+ - | MONTH(DATE('2020-08-26')) | - |-----------------------------| - | 8 | - +-----------------------------+ + +---------------------------+ + | MONTH(DATE('2020-08-26')) | + |---------------------------| + | 8 | + +---------------------------+ os> SELECT MONTH_OF_YEAR(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-------------------------------------+ - | MONTH_OF_YEAR(DATE('2020-08-26')) | - |-------------------------------------| - | 8 | - +-------------------------------------+ + +-----------------------------------+ + | MONTH_OF_YEAR(DATE('2020-08-26')) | + |-----------------------------------| + | 8 | + +-----------------------------------+ MONTHNAME @@ -2352,11 +2352,11 @@ Example:: os> SELECT MONTHNAME(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +---------------------------------+ - | MONTHNAME(DATE('2020-08-26')) | - |---------------------------------| - | August | - +---------------------------------+ + +-------------------------------+ + | MONTHNAME(DATE('2020-08-26')) | + |-------------------------------| + | August | + +-------------------------------+ NOW @@ -2399,11 +2399,11 @@ Example:: os> SELECT PERIOD_ADD(200801, 2), PERIOD_ADD(200801, -12) fetched rows / total rows = 1/1 - +-------------------------+---------------------------+ - | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | - |-------------------------+---------------------------| - | 200803 | 200701 | - +-------------------------+---------------------------+ + +-----------------------+-------------------------+ + | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | + |-----------------------+-------------------------| + | 200803 | 200701 | + +-----------------------+-------------------------+ PERIOD_DIFF @@ -2422,11 +2422,11 @@ Example:: os> SELECT PERIOD_DIFF(200802, 200703), PERIOD_DIFF(200802, 201003) fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+ - | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | - |-------------------------------+-------------------------------| - | 11 | -25 | - +-------------------------------+-------------------------------+ + +-----------------------------+-----------------------------+ + | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | + |-----------------------------+-----------------------------| + | 11 | -25 | + +-----------------------------+-----------------------------+ QUARTER @@ -2445,11 +2445,11 @@ Example:: os> SELECT QUARTER(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-------------------------------+ - | QUARTER(DATE('2020-08-26')) | - |-------------------------------| - | 3 | - +-------------------------------+ + +-----------------------------+ + | QUARTER(DATE('2020-08-26')) | + |-----------------------------| + | 3 | + +-----------------------------+ SEC_TO_TIME @@ -2471,27 +2471,27 @@ Example:: os> SELECT SEC_TO_TIME(3601) fetched rows / total rows = 1/1 - +---------------------+ - | SEC_TO_TIME(3601) | - |---------------------| - | 01:00:01 | - +---------------------+ + +-------------------+ + | SEC_TO_TIME(3601) | + |-------------------| + | 01:00:01 | + +-------------------+ os> SELECT sec_to_time(1234.123); fetched rows / total rows = 1/1 - +-------------------------+ - | sec_to_time(1234.123) | - |-------------------------| - | 00:20:34.123 | - +-------------------------+ + +-----------------------+ + | sec_to_time(1234.123) | + |-----------------------| + | 00:20:34.123 | + +-----------------------+ os> SELECT sec_to_time(NULL); fetched rows / total rows = 1/1 - +---------------------+ - | sec_to_time(NULL) | - |---------------------| - | null | - +---------------------+ + +-------------------+ + | sec_to_time(NULL) | + |-------------------| + | null | + +-------------------+ SECOND @@ -2511,19 +2511,19 @@ Example:: os> SELECT SECOND((TIME '01:02:03')) fetched rows / total rows = 1/1 - +-----------------------------+ - | SECOND((TIME '01:02:03')) | - |-----------------------------| - | 3 | - +-----------------------------+ + +---------------------------+ + | SECOND((TIME '01:02:03')) | + |---------------------------| + | 3 | + +---------------------------+ os> SELECT SECOND_OF_MINUTE(time('01:02:03')) fetched rows / total rows = 1/1 - +--------------------------------------+ - | SECOND_OF_MINUTE(time('01:02:03')) | - |--------------------------------------| - | 3 | - +--------------------------------------+ + +------------------------------------+ + | SECOND_OF_MINUTE(time('01:02:03')) | + |------------------------------------| + | 3 | + +------------------------------------+ STR_TO_DATE @@ -2545,11 +2545,11 @@ Example:: OS> SELECT str_to_date("01,5,2013", "%d,%m,%Y") fetched rows / total rows = 1/1 - +----------------------------------------+ - | str_to_date("01,5,2013", "%d,%m,%Y") | - |----------------------------------------| - | 2013-05-01 00:00:00 | - +----------------------------------------+ + +--------------------------------------+ + | str_to_date("01,5,2013", "%d,%m,%Y") | + |--------------------------------------| + | 2013-05-01 00:00:00 | + +--------------------------------------+ SUBDATE @@ -2579,11 +2579,11 @@ Example:: os> SELECT SUBDATE(DATE('2008-01-02'), INTERVAL 31 DAY) AS `'2008-01-02' - 31d`, SUBDATE(DATE('2020-08-26'), 1) AS `'2020-08-26' - 1`, SUBDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) AS `ts '2020-08-26 01:01:01' - 1` fetched rows / total rows = 1/1 - +----------------------+--------------------+--------------------------------+ - | '2008-01-02' - 31d | '2020-08-26' - 1 | ts '2020-08-26 01:01:01' - 1 | - |----------------------+--------------------+--------------------------------| - | 2007-12-02 00:00:00 | 2020-08-25 | 2020-08-25 01:01:01 | - +----------------------+--------------------+--------------------------------+ + +---------------------+------------------+------------------------------+ + | '2008-01-02' - 31d | '2020-08-26' - 1 | ts '2020-08-26 01:01:01' - 1 | + |---------------------+------------------+------------------------------| + | 2007-12-02 00:00:00 | 2020-08-25 | 2020-08-25 01:01:01 | + +---------------------+------------------+------------------------------+ SUBTIME @@ -2616,35 +2616,35 @@ Example:: os> SELECT SUBTIME(TIME('23:59:59'), DATE('2004-01-01')) AS `'23:59:59' - 0` fetched rows / total rows = 1/1 - +------------------+ - | '23:59:59' - 0 | - |------------------| - | 23:59:59 | - +------------------+ + +----------------+ + | '23:59:59' - 0 | + |----------------| + | 23:59:59 | + +----------------+ os> SELECT SUBTIME(DATE('2004-01-01'), TIME('23:59:59')) AS `'2004-01-01' - '23:59:59'` fetched rows / total rows = 1/1 - +-----------------------------+ - | '2004-01-01' - '23:59:59' | - |-----------------------------| - | 2003-12-31 00:00:01 | - +-----------------------------+ - - os> SELECT SUBTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' - '00:05:42'` - fetched rows / total rows = 1/1 +---------------------------+ - | '10:20:30' - '00:05:42' | + | '2004-01-01' - '23:59:59' | |---------------------------| - | 10:14:48 | + | 2003-12-31 00:00:01 | +---------------------------+ + os> SELECT SUBTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' - '00:05:42'` + fetched rows / total rows = 1/1 + +-------------------------+ + | '10:20:30' - '00:05:42' | + |-------------------------| + | 10:14:48 | + +-------------------------+ + os> SELECT SUBTIME(TIMESTAMP('2007-03-01 10:20:30'), DATETIME('2002-03-04 20:40:50')) AS `'2007-03-01 10:20:30' - '20:40:50'` fetched rows / total rows = 1/1 - +--------------------------------------+ - | '2007-03-01 10:20:30' - '20:40:50' | - |--------------------------------------| - | 2007-02-28 13:39:40 | - +--------------------------------------+ + +------------------------------------+ + | '2007-03-01 10:20:30' - '20:40:50' | + |------------------------------------| + | 2007-02-28 13:39:40 | + +------------------------------------+ SYSDATE @@ -2690,11 +2690,11 @@ Example:: os> SELECT TIME('13:49:00'), TIME('13:49'), TIME(TIMESTAMP('2020-08-26 13:49:00')), TIME('2020-08-26 13:49:00') fetched rows / total rows = 1/1 - +--------------------+-----------------+------------------------------------------+-------------------------------+ - | TIME('13:49:00') | TIME('13:49') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | TIME('2020-08-26 13:49:00') | - |--------------------+-----------------+------------------------------------------+-------------------------------| - | 13:49:00 | 13:49:00 | 13:49:00 | 13:49:00 | - +--------------------+-----------------+------------------------------------------+-------------------------------+ + +------------------+---------------+----------------------------------------+-----------------------------+ + | TIME('13:49:00') | TIME('13:49') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | TIME('2020-08-26 13:49:00') | + |------------------+---------------+----------------------------------------+-----------------------------| + | 13:49:00 | 13:49:00 | 13:49:00 | 13:49:00 | + +------------------+---------------+----------------------------------------+-----------------------------+ TIME_FORMAT ----------- @@ -2744,11 +2744,11 @@ Example:: os> SELECT TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') fetched rows / total rows = 1/1 - +------------------------------------------------------------------------------+ - | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | - |------------------------------------------------------------------------------| - | 012345 13 01 01 14 PM 01:14:15 PM 15 15 13:14:15 | - +------------------------------------------------------------------------------+ + +----------------------------------------------------------------------------+ + | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | + |----------------------------------------------------------------------------| + | 012345 13 01 01 14 PM 01:14:15 PM 15 15 13:14:15 | + +----------------------------------------------------------------------------+ TIME_TO_SEC @@ -2767,11 +2767,11 @@ Example:: os> SELECT TIME_TO_SEC(TIME '22:23:00') fetched rows / total rows = 1/1 - +--------------------------------+ - | TIME_TO_SEC(TIME '22:23:00') | - |--------------------------------| - | 80580 | - +--------------------------------+ + +------------------------------+ + | TIME_TO_SEC(TIME '22:23:00') | + |------------------------------| + | 80580 | + +------------------------------+ TIMEDIFF @@ -2790,11 +2790,11 @@ Example:: os> SELECT TIMEDIFF('23:59:59', '13:00:00') fetched rows / total rows = 1/1 - +------------------------------------+ - | TIMEDIFF('23:59:59', '13:00:00') | - |------------------------------------| - | 10:59:59 | - +------------------------------------+ + +----------------------------------+ + | TIMEDIFF('23:59:59', '13:00:00') | + |----------------------------------| + | 10:59:59 | + +----------------------------------+ TIMESTAMP @@ -2818,11 +2818,11 @@ Example:: os> SELECT TIMESTAMP('2020-08-26 13:49:00'), TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) fetched rows / total rows = 1/1 - +------------------------------------+------------------------------------------------------+ - | TIMESTAMP('2020-08-26 13:49:00') | TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | - |------------------------------------+------------------------------------------------------| - | 2020-08-26 13:49:00 | 2020-08-27 02:04:42 | - +------------------------------------+------------------------------------------------------+ + +----------------------------------+----------------------------------------------------+ + | TIMESTAMP('2020-08-26 13:49:00') | TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | + |----------------------------------+----------------------------------------------------| + | 2020-08-26 13:49:00 | 2020-08-27 02:04:42 | + +----------------------------------+----------------------------------------------------+ TIMESTAMPADD @@ -2842,11 +2842,11 @@ Examples:: os> SELECT TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00'), TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') fetched rows / total rows = 1/1 - +------------------------------------------------+----------------------------------------------------+ - | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | - |------------------------------------------------+----------------------------------------------------| - | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | - +------------------------------------------------+----------------------------------------------------+ + +----------------------------------------------+--------------------------------------------------+ + | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | + |----------------------------------------------+--------------------------------------------------| + | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | + +----------------------------------------------+--------------------------------------------------+ TIMESTAMPDIFF @@ -2867,11 +2867,11 @@ Examples:: os> SELECT TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00'), TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) fetched rows / total rows = 1/1 - +---------------------------------------------------------------------+-------------------------------------------------------------+ - | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | - |---------------------------------------------------------------------+-------------------------------------------------------------| - | 4 | -23 | - +---------------------------------------------------------------------+-------------------------------------------------------------+ + +-------------------------------------------------------------------+-----------------------------------------------------------+ + | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | + |-------------------------------------------------------------------+-----------------------------------------------------------| + | 4 | -23 | + +-------------------------------------------------------------------+-----------------------------------------------------------+ TO_DAYS @@ -2890,11 +2890,11 @@ Example:: os> SELECT TO_DAYS(DATE '2008-10-07') fetched rows / total rows = 1/1 - +------------------------------+ - | TO_DAYS(DATE '2008-10-07') | - |------------------------------| - | 733687 | - +------------------------------+ + +----------------------------+ + | TO_DAYS(DATE '2008-10-07') | + |----------------------------| + | 733687 | + +----------------------------+ TO_SECONDS @@ -2914,11 +2914,11 @@ Example:: os> SELECT TO_SECONDS(DATE '2008-10-07'), TO_SECONDS(950228) fetched rows / total rows = 1/1 - +---------------------------------+----------------------+ - | TO_SECONDS(DATE '2008-10-07') | TO_SECONDS(950228) | - |---------------------------------+----------------------| - | 63390556800 | 62961148800 | - +---------------------------------+----------------------+ + +-------------------------------+--------------------+ + | TO_SECONDS(DATE '2008-10-07') | TO_SECONDS(950228) | + |-------------------------------+--------------------| + | 63390556800 | 62961148800 | + +-------------------------------+--------------------+ UNIX_TIMESTAMP @@ -2940,19 +2940,19 @@ Examples:: os> select UNIX_TIMESTAMP(20771122143845) fetched rows / total rows = 1/1 - +----------------------------------+ - | UNIX_TIMESTAMP(20771122143845) | - |----------------------------------| - | 3404817525.0 | - +----------------------------------+ + +--------------------------------+ + | UNIX_TIMESTAMP(20771122143845) | + |--------------------------------| + | 3404817525.0 | + +--------------------------------+ os> select UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) fetched rows / total rows = 1/1 - +----------------------------------------------------+ - | UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | - |----------------------------------------------------| - | 848077542.0 | - +----------------------------------------------------+ + +--------------------------------------------------+ + | UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | + |--------------------------------------------------| + | 848077542.0 | + +--------------------------------------------------+ UTC_DATE @@ -2971,11 +2971,11 @@ Example:: > SELECT UTC_DATE(); fetched rows / total rows = 1/1 - +--------------+ - | utc_date() | - |--------------| - | 2022-10-03 | - +--------------+ + +------------+ + | utc_date() | + |------------| + | 2022-10-03 | + +------------+ UTC_TIME @@ -2994,11 +2994,11 @@ Example:: > SELECT UTC_TIME(); fetched rows / total rows = 1/1 - +--------------+ - | utc_time() | - |--------------| - | 17:54:27 | - +--------------+ + +------------+ + | utc_time() | + |------------| + | 17:54:27 | + +------------+ UTC_TIMESTAMP @@ -3083,11 +3083,11 @@ Example:: os> SELECT WEEK(DATE('2008-02-20')), WEEK(DATE('2008-02-20'), 1) fetched rows / total rows = 1/1 - +----------------------------+-------------------------------+ - | WEEK(DATE('2008-02-20')) | WEEK(DATE('2008-02-20'), 1) | - |----------------------------+-------------------------------| - | 7 | 8 | - +----------------------------+-------------------------------+ + +--------------------------+-----------------------------+ + | WEEK(DATE('2008-02-20')) | WEEK(DATE('2008-02-20'), 1) | + |--------------------------+-----------------------------| + | 7 | 8 | + +--------------------------+-----------------------------+ WEEKDAY @@ -3108,11 +3108,11 @@ Example:: os> SELECT weekday('2020-08-26'), weekday('2020-08-27') fetched rows / total rows = 1/1 - +-------------------------+-------------------------+ - | weekday('2020-08-26') | weekday('2020-08-27') | - |-------------------------+-------------------------| - | 2 | 3 | - +-------------------------+-------------------------+ + +-----------------------+-----------------------+ + | weekday('2020-08-26') | weekday('2020-08-27') | + |-----------------------+-----------------------| + | 2 | 3 | + +-----------------------+-----------------------+ WEEK_OF_YEAR @@ -3132,11 +3132,11 @@ Example:: os> SELECT WEEK_OF_YEAR(DATE('2008-02-20')), WEEK_OF_YEAR(DATE('2008-02-20'), 1) fetched rows / total rows = 1/1 - +------------------------------------+---------------------------------------+ - | WEEK_OF_YEAR(DATE('2008-02-20')) | WEEK_OF_YEAR(DATE('2008-02-20'), 1) | - |------------------------------------+---------------------------------------| - | 7 | 8 | - +------------------------------------+---------------------------------------+ + +----------------------------------+-------------------------------------+ + | WEEK_OF_YEAR(DATE('2008-02-20')) | WEEK_OF_YEAR(DATE('2008-02-20'), 1) | + |----------------------------------+-------------------------------------| + | 7 | 8 | + +----------------------------------+-------------------------------------+ WEEKOFYEAR @@ -3156,11 +3156,11 @@ Example:: os> SELECT WEEKOFYEAR(DATE('2008-02-20')), WEEKOFYEAR(DATE('2008-02-20'), 1) fetched rows / total rows = 1/1 - +----------------------------------+-------------------------------------+ - | WEEKOFYEAR(DATE('2008-02-20')) | WEEKOFYEAR(DATE('2008-02-20'), 1) | - |----------------------------------+-------------------------------------| - | 7 | 8 | - +----------------------------------+-------------------------------------+ + +--------------------------------+-----------------------------------+ + | WEEKOFYEAR(DATE('2008-02-20')) | WEEKOFYEAR(DATE('2008-02-20'), 1) | + |--------------------------------+-----------------------------------| + | 7 | 8 | + +--------------------------------+-----------------------------------+ YEAR @@ -3179,11 +3179,11 @@ Example:: os> SELECT YEAR(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +----------------------------+ - | YEAR(DATE('2020-08-26')) | - |----------------------------| - | 2020 | - +----------------------------+ + +--------------------------+ + | YEAR(DATE('2020-08-26')) | + |--------------------------| + | 2020 | + +--------------------------+ YEARWEEK @@ -3202,11 +3202,11 @@ Example:: os> SELECT YEARWEEK('2020-08-26'), YEARWEEK('2019-01-05', 0) fetched rows / total rows = 1/1 - +--------------------------+-----------------------------+ - | YEARWEEK('2020-08-26') | YEARWEEK('2019-01-05', 0) | - |--------------------------+-----------------------------| - | 202034 | 201852 | - +--------------------------+-----------------------------+ + +------------------------+---------------------------+ + | YEARWEEK('2020-08-26') | YEARWEEK('2019-01-05', 0) | + |------------------------+---------------------------| + | 202034 | 201852 | + +------------------------+---------------------------+ String Functions @@ -3228,11 +3228,11 @@ Example:: os> SELECT ASCII('hello') fetched rows / total rows = 1/1 - +------------------+ - | ASCII('hello') | - |------------------| - | 104 | - +------------------+ + +----------------+ + | ASCII('hello') | + |----------------| + | 104 | + +----------------+ CONCAT @@ -3251,11 +3251,11 @@ Example:: os> SELECT CONCAT('hello ', 'whole ', 'world', '!'), CONCAT('hello', 'world'), CONCAT('hello', null) fetched rows / total rows = 1/1 - +--------------------------------------------+----------------------------+-------------------------+ - | CONCAT('hello ', 'whole ', 'world', '!') | CONCAT('hello', 'world') | CONCAT('hello', null) | - |--------------------------------------------+----------------------------+-------------------------| - | hello whole world! | helloworld | null | - +--------------------------------------------+----------------------------+-------------------------+ + +------------------------------------------+--------------------------+-----------------------+ + | CONCAT('hello ', 'whole ', 'world', '!') | CONCAT('hello', 'world') | CONCAT('hello', null) | + |------------------------------------------+--------------------------+-----------------------| + | hello whole world! | helloworld | null | + +------------------------------------------+--------------------------+-----------------------+ CONCAT_WS @@ -3274,11 +3274,11 @@ Example:: os> SELECT CONCAT_WS(',', 'hello', 'world') fetched rows / total rows = 1/1 - +------------------------------------+ - | CONCAT_WS(',', 'hello', 'world') | - |------------------------------------| - | hello,world | - +------------------------------------+ + +----------------------------------+ + | CONCAT_WS(',', 'hello', 'world') | + |----------------------------------| + | hello,world | + +----------------------------------+ LEFT @@ -3294,11 +3294,11 @@ Example:: os> SELECT LEFT('helloworld', 5), LEFT('HELLOWORLD', 0) fetched rows / total rows = 1/1 - +-------------------------+-------------------------+ - | LEFT('helloworld', 5) | LEFT('HELLOWORLD', 0) | - |-------------------------+-------------------------| - | hello | | - +-------------------------+-------------------------+ + +-----------------------+-----------------------+ + | LEFT('helloworld', 5) | LEFT('HELLOWORLD', 0) | + |-----------------------+-----------------------| + | hello | | + +-----------------------+-----------------------+ LENGTH @@ -3317,11 +3317,11 @@ Example:: os> SELECT LENGTH('helloworld') fetched rows / total rows = 1/1 - +------------------------+ - | LENGTH('helloworld') | - |------------------------| - | 10 | - +------------------------+ + +----------------------+ + | LENGTH('helloworld') | + |----------------------| + | 10 | + +----------------------+ LOCATE @@ -3343,11 +3343,11 @@ Example:: os> SELECT LOCATE('world', 'helloworld'), LOCATE('world', 'helloworldworld', 7) fetched rows / total rows = 1/1 - +---------------------------------+-----------------------------------------+ - | LOCATE('world', 'helloworld') | LOCATE('world', 'helloworldworld', 7) | - |---------------------------------+-----------------------------------------| - | 6 | 11 | - +---------------------------------+-----------------------------------------+ + +-------------------------------+---------------------------------------+ + | LOCATE('world', 'helloworld') | LOCATE('world', 'helloworldworld', 7) | + |-------------------------------+---------------------------------------| + | 6 | 11 | + +-------------------------------+---------------------------------------+ LOWER @@ -3366,11 +3366,11 @@ Example:: os> SELECT LOWER('helloworld'), LOWER('HELLOWORLD') fetched rows / total rows = 1/1 - +-----------------------+-----------------------+ - | LOWER('helloworld') | LOWER('HELLOWORLD') | - |-----------------------+-----------------------| - | helloworld | helloworld | - +-----------------------+-----------------------+ + +---------------------+---------------------+ + | LOWER('helloworld') | LOWER('HELLOWORLD') | + |---------------------+---------------------| + | helloworld | helloworld | + +---------------------+---------------------+ LTRIM @@ -3389,11 +3389,11 @@ Example:: os> SELECT LTRIM(' hello'), LTRIM('hello ') fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | LTRIM(' hello') | LTRIM('hello ') | - |---------------------+---------------------| - | hello | hello | - +---------------------+---------------------+ + +-------------------+-------------------+ + | LTRIM(' hello') | LTRIM('hello ') | + |-------------------+-------------------| + | hello | hello | + +-------------------+-------------------+ POSITION @@ -3414,11 +3414,11 @@ Example:: os> SELECT POSITION('world' IN 'helloworld'), POSITION('invalid' IN 'helloworld'); fetched rows / total rows = 1/1 - +-------------------------------------+---------------------------------------+ - | POSITION('world' IN 'helloworld') | POSITION('invalid' IN 'helloworld') | - |-------------------------------------+---------------------------------------| - | 6 | 0 | - +-------------------------------------+---------------------------------------+ + +-----------------------------------+-------------------------------------+ + | POSITION('world' IN 'helloworld') | POSITION('invalid' IN 'helloworld') | + |-----------------------------------+-------------------------------------| + | 6 | 0 | + +-----------------------------------+-------------------------------------+ REPLACE @@ -3437,11 +3437,11 @@ Example:: os> SELECT REPLACE('Hello World!', 'World', 'OpenSearch') fetched rows / total rows = 1/1 - +--------------------------------------------------+ - | REPLACE('Hello World!', 'World', 'OpenSearch') | - |--------------------------------------------------| - | Hello OpenSearch! | - +--------------------------------------------------+ + +------------------------------------------------+ + | REPLACE('Hello World!', 'World', 'OpenSearch') | + |------------------------------------------------| + | Hello OpenSearch! | + +------------------------------------------------+ REVERSE @@ -3460,11 +3460,11 @@ Example:: os> SELECT REVERSE('abcde'), REVERSE(null) fetched rows / total rows = 1/1 - +--------------------+-----------------+ - | REVERSE('abcde') | REVERSE(null) | - |--------------------+-----------------| - | edcba | null | - +--------------------+-----------------+ + +------------------+---------------+ + | REVERSE('abcde') | REVERSE(null) | + |------------------+---------------| + | edcba | null | + +------------------+---------------+ RIGHT @@ -3483,11 +3483,11 @@ Example:: os> SELECT RIGHT('helloworld', 5), RIGHT('HELLOWORLD', 0) fetched rows / total rows = 1/1 - +--------------------------+--------------------------+ - | RIGHT('helloworld', 5) | RIGHT('HELLOWORLD', 0) | - |--------------------------+--------------------------| - | world | | - +--------------------------+--------------------------+ + +------------------------+------------------------+ + | RIGHT('helloworld', 5) | RIGHT('HELLOWORLD', 0) | + |------------------------+------------------------| + | world | | + +------------------------+------------------------+ RTRIM @@ -3506,11 +3506,11 @@ Example:: os> SELECT RTRIM(' hello'), RTRIM('hello ') fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | RTRIM(' hello') | RTRIM('hello ') | - |---------------------+---------------------| - | hello | hello | - +---------------------+---------------------+ + +-------------------+-------------------+ + | RTRIM(' hello') | RTRIM('hello ') | + |-------------------+-------------------| + | hello | hello | + +-------------------+-------------------+ SUBSTRING @@ -3531,11 +3531,11 @@ Example:: os> SELECT SUBSTRING('helloworld', 5), SUBSTRING('helloworld', 5, 3) fetched rows / total rows = 1/1 - +------------------------------+---------------------------------+ - | SUBSTRING('helloworld', 5) | SUBSTRING('helloworld', 5, 3) | - |------------------------------+---------------------------------| - | oworld | owo | - +------------------------------+---------------------------------+ + +----------------------------+-------------------------------+ + | SUBSTRING('helloworld', 5) | SUBSTRING('helloworld', 5, 3) | + |----------------------------+-------------------------------| + | oworld | owo | + +----------------------------+-------------------------------+ TRIM @@ -3552,11 +3552,11 @@ Example:: os> SELECT TRIM(' hello'), TRIM('hello ') fetched rows / total rows = 1/1 - +--------------------+--------------------+ - | TRIM(' hello') | TRIM('hello ') | - |--------------------+--------------------| - | hello | hello | - +--------------------+--------------------+ + +------------------+------------------+ + | TRIM(' hello') | TRIM('hello ') | + |------------------+------------------| + | hello | hello | + +------------------+------------------+ UPPER @@ -3575,11 +3575,11 @@ Example:: os> SELECT UPPER('helloworld'), UPPER('HELLOWORLD') fetched rows / total rows = 1/1 - +-----------------------+-----------------------+ - | UPPER('helloworld') | UPPER('HELLOWORLD') | - |-----------------------+-----------------------| - | HELLOWORLD | HELLOWORLD | - +-----------------------+-----------------------+ + +---------------------+---------------------+ + | UPPER('helloworld') | UPPER('HELLOWORLD') | + |---------------------+---------------------| + | HELLOWORLD | HELLOWORLD | + +---------------------+---------------------+ @@ -3602,31 +3602,31 @@ Example One:: os> SELECT IFNULL(123, 321), IFNULL(321, 123) fetched rows / total rows = 1/1 - +--------------------+--------------------+ - | IFNULL(123, 321) | IFNULL(321, 123) | - |--------------------+--------------------| - | 123 | 321 | - +--------------------+--------------------+ + +------------------+------------------+ + | IFNULL(123, 321) | IFNULL(321, 123) | + |------------------+------------------| + | 123 | 321 | + +------------------+------------------+ Example Two:: os> SELECT IFNULL(321, 1/0), IFNULL(1/0, 123) fetched rows / total rows = 1/1 - +--------------------+--------------------+ - | IFNULL(321, 1/0) | IFNULL(1/0, 123) | - |--------------------+--------------------| - | 321 | 123 | - +--------------------+--------------------+ + +------------------+------------------+ + | IFNULL(321, 1/0) | IFNULL(1/0, 123) | + |------------------+------------------| + | 321 | 123 | + +------------------+------------------+ Example Three:: os> SELECT IFNULL(1/0, 1/0) fetched rows / total rows = 1/1 - +--------------------+ - | IFNULL(1/0, 1/0) | - |--------------------| - | null | - +--------------------+ + +------------------+ + | IFNULL(1/0, 1/0) | + |------------------| + | null | + +------------------+ NULLIF @@ -3645,11 +3645,11 @@ Example:: os> SELECT NULLIF(123, 123), NULLIF(321, 123), NULLIF(1/0, 321), NULLIF(321, 1/0), NULLIF(1/0, 1/0) fetched rows / total rows = 1/1 - +--------------------+--------------------+--------------------+--------------------+--------------------+ - | NULLIF(123, 123) | NULLIF(321, 123) | NULLIF(1/0, 321) | NULLIF(321, 1/0) | NULLIF(1/0, 1/0) | - |--------------------+--------------------+--------------------+--------------------+--------------------| - | null | 321 | null | 321 | null | - +--------------------+--------------------+--------------------+--------------------+--------------------+ + +------------------+------------------+------------------+------------------+------------------+ + | NULLIF(123, 123) | NULLIF(321, 123) | NULLIF(1/0, 321) | NULLIF(321, 1/0) | NULLIF(1/0, 1/0) | + |------------------+------------------+------------------+------------------+------------------| + | null | 321 | null | 321 | null | + +------------------+------------------+------------------+------------------+------------------+ ISNULL @@ -3668,11 +3668,11 @@ Example:: os> SELECT ISNULL(1/0), ISNULL(123) fetched rows / total rows = 1/1 - +---------------+---------------+ - | ISNULL(1/0) | ISNULL(123) | - |---------------+---------------| - | True | False | - +---------------+---------------+ + +-------------+-------------+ + | ISNULL(1/0) | ISNULL(123) | + |-------------+-------------| + | True | False | + +-------------+-------------+ IF @@ -3693,19 +3693,19 @@ Example:: os> SELECT IF(100 > 200, '100', '200') fetched rows / total rows = 1/1 - +-------------------------------+ - | IF(100 > 200, '100', '200') | - |-------------------------------| - | 200 | - +-------------------------------+ + +-----------------------------+ + | IF(100 > 200, '100', '200') | + |-----------------------------| + | 200 | + +-----------------------------+ os> SELECT IF(200 > 100, '100', '200') fetched rows / total rows = 1/1 - +-------------------------------+ - | IF(200 > 100, '100', '200') | - |-------------------------------| - | 100 | - +-------------------------------+ + +-----------------------------+ + | IF(200 > 100, '100', '200') | + |-----------------------------| + | 100 | + +-----------------------------+ CASE @@ -3760,11 +3760,11 @@ Here are examples for simple case syntax:: ... ELSE TRIM(' Absolute three ') ... END AS func_result; fetched rows / total rows = 1/1 - +---------------+-------------------+----------------+ - | simple_case | func_case_value | func_result | - |---------------+-------------------+----------------| - | One | Absolute two | Absolute three | - +---------------+-------------------+----------------+ + +-------------+-----------------+----------------+ + | simple_case | func_case_value | func_result | + |-------------+-----------------+----------------| + | One | Absolute two | Absolute three | + +-------------+-----------------+----------------+ Here are examples for searched case syntax:: @@ -3780,11 +3780,11 @@ Here are examples for searched case syntax:: ... WHEN 'hello' = 'world' THEN 'Hello' ... END AS no_else; fetched rows / total rows = 1/1 - +-----------------+------------------+-----------+ - | single_search | multi_searches | no_else | - |-----------------+------------------+-----------| - | One | Hello | null | - +-----------------+------------------+-----------+ + +---------------+----------------+---------+ + | single_search | multi_searches | no_else | + |---------------+----------------+---------| + | One | Hello | null | + +---------------+----------------+---------+ RELEVANCE @@ -3819,22 +3819,22 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> SELECT lastname, address FROM accounts WHERE match(address, 'Street'); fetched rows / total rows = 2/2 - +------------+--------------------+ - | lastname | address | - |------------+--------------------| - | Bond | 671 Bristol Street | - | Bates | 789 Madison Street | - +------------+--------------------+ + +----------+--------------------+ + | lastname | address | + |----------+--------------------| + | Bond | 671 Bristol Street | + | Bates | 789 Madison Street | + +----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> SELECT lastname FROM accounts WHERE match(firstname, 'Hattie', operator='AND', boost=2.0); fetched rows / total rows = 1/1 - +------------+ - | lastname | - |------------| - | Bond | - +------------+ + +----------+ + | lastname | + |----------| + | Bond | + +----------+ MATCHQUERY @@ -3849,32 +3849,32 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> SELECT lastname, address FROM accounts WHERE matchquery(address, 'Street'); fetched rows / total rows = 2/2 - +------------+--------------------+ - | lastname | address | - |------------+--------------------| - | Bond | 671 Bristol Street | - | Bates | 789 Madison Street | - +------------+--------------------+ + +----------+--------------------+ + | lastname | address | + |----------+--------------------| + | Bond | 671 Bristol Street | + | Bates | 789 Madison Street | + +----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> SELECT lastname FROM accounts WHERE matchquery(firstname, 'Hattie', operator='AND', boost=2.0); fetched rows / total rows = 1/1 - +------------+ - | lastname | - |------------| - | Bond | - +------------+ + +----------+ + | lastname | + |----------| + | Bond | + +----------+ The matchquery function also supports an alternative syntax:: os> SELECT firstname FROM accounts WHERE firstname = matchquery('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ MATCH_QUERY @@ -3889,32 +3889,32 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> SELECT lastname, address FROM accounts WHERE match_query(address, 'Street'); fetched rows / total rows = 2/2 - +------------+--------------------+ - | lastname | address | - |------------+--------------------| - | Bond | 671 Bristol Street | - | Bates | 789 Madison Street | - +------------+--------------------+ + +----------+--------------------+ + | lastname | address | + |----------+--------------------| + | Bond | 671 Bristol Street | + | Bates | 789 Madison Street | + +----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> SELECT lastname FROM accounts WHERE match_query(firstname, 'Hattie', operator='AND', boost=2.0); fetched rows / total rows = 1/1 - +------------+ - | lastname | - |------------| - | Bond | - +------------+ + +----------+ + | lastname | + |----------| + | Bond | + +----------+ The match_query function also supports an alternative syntax:: os> SELECT firstname FROM accounts WHERE firstname = match_query('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ MATCH_PHRASE @@ -3959,19 +3959,19 @@ The match_phrase function also supports an alternative syntax:: os> SELECT firstname FROM accounts WHERE firstname = match_phrase('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ os> SELECT firstname FROM accounts WHERE firstname = matchphrase('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ MATCH_BOOL_PREFIX @@ -3998,22 +3998,22 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> SELECT firstname, address FROM accounts WHERE match_bool_prefix(address, 'Bristol Stre'); fetched rows / total rows = 2/2 - +-------------+--------------------+ - | firstname | address | - |-------------+--------------------| - | Hattie | 671 Bristol Street | - | Nanette | 789 Madison Street | - +-------------+--------------------+ + +-----------+--------------------+ + | firstname | address | + |-----------+--------------------| + | Hattie | 671 Bristol Street | + | Nanette | 789 Madison Street | + +-----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> SELECT firstname, address FROM accounts WHERE match_bool_prefix(address, 'Bristol Street', minimum_should_match=2); fetched rows / total rows = 1/1 - +-------------+--------------------+ - | firstname | address | - |-------------+--------------------| - | Hattie | 671 Bristol Street | - +-------------+--------------------+ + +-----------+--------------------+ + | firstname | address | + |-----------+--------------------| + | Hattie | 671 Bristol Street | + +-----------+--------------------+ MATCH_PHRASE_PREFIX @@ -4100,40 +4100,40 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> select id, title, author from books where multi_match(['title'], 'Pooh House'); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> select id, title, author from books where multi_match(['title'], 'Pooh House', operator='AND', analyzer=default); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ The multi_match function also supports an alternative syntax:: os> SELECT firstname FROM accounts WHERE firstname = multi_match('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ os> SELECT firstname FROM accounts WHERE firstname = multimatch('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ SIMPLE_QUERY_STRING @@ -4170,22 +4170,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> select id, title, author from books where simple_query_string(['title'], 'Pooh House'); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> select id, title, author from books where simple_query_string(['title'], 'Pooh House', flags='ALL', default_operator='AND'); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ QUERY_STRING @@ -4232,22 +4232,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> select id, title, author from books where query_string(['title'], 'Pooh House'); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> select id, title, author from books where query_string(['title'], 'Pooh House', default_operator='AND'); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ QUERY @@ -4294,22 +4294,22 @@ Example with only ``query_expressions``, and all other parameters are set defaul os> select id, title, author from books where query('title:Pooh House'); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> select id, title, author from books where query('title:Pooh House', default_operator='AND'); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ SCORE @@ -4339,20 +4339,20 @@ Example boosting score:: os> select id, title, author, _score from books where score(query('title:Pooh House', default_operator='AND'), 2.0); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+-----------+ - | id | title | author | _score | - |------+--------------------------+----------------------+-----------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | 1.5884793 | - +------+--------------------------+----------------------+-----------+ + +----+--------------------------+----------------------+-----------+ + | id | title | author | _score | + |----+--------------------------+----------------------+-----------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | 1.5884793 | + +----+--------------------------+----------------------+-----------+ os> select id, title, author, _score from books where score(query('title:Pooh House', default_operator='AND'), 5.0) OR score(query('title:Winnie', default_operator='AND'), 1.5); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+-----------+ - | id | title | author | _score | - |------+--------------------------+----------------------+-----------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | 3.9711983 | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | 1.1581701 | - +------+--------------------------+----------------------+-----------+ + +----+--------------------------+----------------------+-----------+ + | id | title | author | _score | + |----+--------------------------+----------------------+-----------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | 3.9711983 | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | 1.1581701 | + +----+--------------------------+----------------------+-----------+ HIGHLIGHT @@ -4451,45 +4451,45 @@ Example with ``field`` and ``path`` parameters:: os> SELECT nested(message.info, message) FROM nested; fetched rows / total rows = 2/2 - +---------------------------------+ - | nested(message.info, message) | - |---------------------------------| - | a | - | b | - +---------------------------------+ + +-------------------------------+ + | nested(message.info, message) | + |-------------------------------| + | a | + | b | + +-------------------------------+ Example with ``field.*`` used in SELECT clause:: os> SELECT nested(message.*) FROM nested; fetched rows / total rows = 2/2 - +--------------------------+-----------------------------+------------------------+ - | nested(message.author) | nested(message.dayOfWeek) | nested(message.info) | - |--------------------------+-----------------------------+------------------------| - | e | 1 | a | - | f | 2 | b | - +--------------------------+-----------------------------+------------------------+ + +------------------------+---------------------------+----------------------+ + | nested(message.author) | nested(message.dayOfWeek) | nested(message.info) | + |------------------------+---------------------------+----------------------| + | e | 1 | a | + | f | 2 | b | + +------------------------+---------------------------+----------------------+ Example with ``field`` and ``path`` parameters in the SELECT and WHERE clause:: os> SELECT nested(message.info, message) FROM nested WHERE nested(message.info, message) = 'b'; fetched rows / total rows = 1/1 - +---------------------------------+ - | nested(message.info, message) | - |---------------------------------| - | b | - +---------------------------------+ + +-------------------------------+ + | nested(message.info, message) | + |-------------------------------| + | b | + +-------------------------------+ Example with ``field`` and ``path`` parameters in the SELECT and ORDER BY clause:: os> SELECT nested(message.info, message) FROM nested ORDER BY nested(message.info, message) DESC; fetched rows / total rows = 2/2 - +---------------------------------+ - | nested(message.info, message) | - |---------------------------------| - | b | - | a | - +---------------------------------+ + +-------------------------------+ + | nested(message.info, message) | + |-------------------------------| + | b | + | a | + +-------------------------------+ System Functions @@ -4511,9 +4511,9 @@ Example:: os> select typeof(DATE('2008-04-14')) as `typeof(date)`, typeof(1) as `typeof(int)`, typeof(now()) as `typeof(now())`, typeof(accounts) as `typeof(column)` from people fetched rows / total rows = 1/1 - +----------------+---------------+-----------------+------------------+ - | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | - |----------------+---------------+-----------------+------------------| - | DATE | INTEGER | DATETIME | OBJECT | - +----------------+---------------+-----------------+------------------+ + +--------------+-------------+---------------+----------------+ + | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | + |--------------+-------------+---------------+----------------| + | DATE | INTEGER | DATETIME | OBJECT | + +--------------+-------------+---------------+----------------+ diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index 149d52d3f9..2b8e6e52b1 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -38,19 +38,19 @@ SQL query:: os> SHOW TABLES LIKE '%' fetched rows / total rows = 9/9 - +----------------+---------------+-----------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ - | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | - |----------------+---------------+-----------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------| - | docTestCluster | null | .ql-datasources | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | account2 | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | apache | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | books | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | wildcard | BASE TABLE | null | null | null | null | null | null | - +----------------+---------------+-----------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ + +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | + |----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------| + | docTestCluster | null | .ql-datasources | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | account2 | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | apache | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | books | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | wildcard | BASE TABLE | null | null | null | null | null | null | + +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ Example 2: Show Specific Index Information ------------------------------------------ @@ -59,19 +59,14 @@ Here is an example that searches metadata for index name prefixed by 'acc'. Besi SQL query:: - POST /_plugins/_sql - { - "query" : "SHOW TABLES LIKE acc%" - } - -Result set: - -+---------+-----------+----------+----------+-------+--------+----------+---------+-------------------------+--------------+ -|TABLE_CAT|TABLE_SCHEM|TABLE_NAME|TABLE_TYPE|REMARKS|TYPE_CAT|TYPE_SCHEM|TYPE_NAME|SELF_REFERENCING_COL_NAME|REF_GENERATION| -+=========+===========+==========+==========+=======+========+==========+=========+=========================+==============+ -|integTest| null| accounts|BASE TABLE| null| null| null| null| null| null| -+---------+-----------+----------+----------+-------+--------+----------+---------+-------------------------+--------------+ - + os> SHOW TABLES LIKE "acc%" + fetched rows / total rows = 2/2 + +----------------+-------------+------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | + |----------------+-------------+------------+------------+---------+----------+------------+-----------+---------------------------+----------------| + | docTestCluster | null | account2 | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | + +----------------+-------------+------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ Example 3: Describe Index Fields Information -------------------------------------------- @@ -80,10 +75,23 @@ Example 3: Describe Index Fields Information SQL query:: - POST /_plugins/_sql - { - "query" : "DESCRIBE TABLES LIKE accounts" - } + os> DESCRIBE TABLES LIKE 'accounts' + fetched rows / total rows = 11/11 + +----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | + |----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------| + | docTestCluster | null | accounts | account_number | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 0 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | address | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 2 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | balance | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 3 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | gender | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 4 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | city | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 5 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | employer | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 6 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | state | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 7 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | age | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 8 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | email | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 9 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | + +----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ Result set: @@ -114,3 +122,11 @@ Result set: +---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ + os> DESCRIBE TABLES LIKE "accounts" COLUMNS LIKE "%name" + fetched rows / total rows = 2/2 + +----------------+-------------+------------+-------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | + |----------------+-------------+------------+-------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------| + | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | + +----------------+-------------+------------+-------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ diff --git a/docs/user/dql/window.rst b/docs/user/dql/window.rst index feb2aaa44e..f0c53da055 100644 --- a/docs/user/dql/window.rst +++ b/docs/user/dql/window.rst @@ -53,14 +53,14 @@ Here is an example for ``COUNT`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | cnt | - |----------+-----------+-------| - | F | 32838 | 1 | - | M | 4180 | 1 | - | M | 5686 | 2 | - | M | 39225 | 3 | - +----------+-----------+-------+ + +--------+---------+-----+ + | gender | balance | cnt | + |--------+---------+-----| + | F | 32838 | 1 | + | M | 4180 | 1 | + | M | 5686 | 2 | + | M | 39225 | 3 | + +--------+---------+-----+ MIN --- @@ -74,14 +74,14 @@ Here is an example for ``MIN`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | cnt | - |----------+-----------+-------| - | F | 32838 | 32838 | - | M | 4180 | 4180 | - | M | 5686 | 4180 | - | M | 39225 | 4180 | - +----------+-----------+-------+ + +--------+---------+-------+ + | gender | balance | cnt | + |--------+---------+-------| + | F | 32838 | 32838 | + | M | 4180 | 4180 | + | M | 5686 | 4180 | + | M | 39225 | 4180 | + +--------+---------+-------+ MAX --- @@ -95,14 +95,14 @@ Here is an example for ``MAX`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | cnt | - |----------+-----------+-------| - | F | 32838 | 32838 | - | M | 4180 | 4180 | - | M | 5686 | 5686 | - | M | 39225 | 39225 | - +----------+-----------+-------+ + +--------+---------+-------+ + | gender | balance | cnt | + |--------+---------+-------| + | F | 32838 | 32838 | + | M | 4180 | 4180 | + | M | 5686 | 5686 | + | M | 39225 | 39225 | + +--------+---------+-------+ AVG --- @@ -116,14 +116,14 @@ Here is an example for ``AVG`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+--------------------+ - | gender | balance | cnt | - |----------+-----------+--------------------| - | F | 32838 | 32838.0 | - | M | 4180 | 4180.0 | - | M | 5686 | 4933.0 | - | M | 39225 | 16363.666666666666 | - +----------+-----------+--------------------+ + +--------+---------+--------------------+ + | gender | balance | cnt | + |--------+---------+--------------------| + | F | 32838 | 32838.0 | + | M | 4180 | 4180.0 | + | M | 5686 | 4933.0 | + | M | 39225 | 16363.666666666666 | + +--------+---------+--------------------+ SUM --- @@ -137,14 +137,14 @@ Here is an example for ``SUM`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | cnt | - |----------+-----------+-------| - | F | 32838 | 32838 | - | M | 4180 | 4180 | - | M | 5686 | 9866 | - | M | 39225 | 49091 | - +----------+-----------+-------+ + +--------+---------+-------+ + | gender | balance | cnt | + |--------+---------+-------| + | F | 32838 | 32838 | + | M | 4180 | 4180 | + | M | 5686 | 9866 | + | M | 39225 | 49091 | + +--------+---------+-------+ STDDEV_POP ---------- @@ -158,14 +158,14 @@ Here is an example for ``STDDEV_POP`` function:: ... ) AS val ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+--------------------+ - | gender | balance | val | - |----------+-----------+--------------------| - | F | 32838 | 0.0 | - | M | 4180 | 0.0 | - | M | 5686 | 753.0 | - | M | 39225 | 16177.091422406222 | - +----------+-----------+--------------------+ + +--------+---------+--------------------+ + | gender | balance | val | + |--------+---------+--------------------| + | F | 32838 | 0.0 | + | M | 4180 | 0.0 | + | M | 5686 | 753.0 | + | M | 39225 | 16177.091422406222 | + +--------+---------+--------------------+ STDDEV_SAMP ----------- @@ -179,14 +179,14 @@ Here is an example for ``STDDEV_SAMP`` function:: ... ) AS val ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+--------------------+ - | gender | balance | val | - |----------+-----------+--------------------| - | F | 32838 | 0.0 | - | M | 4180 | 0.0 | - | M | 5686 | 1064.9028124669405 | - | M | 39225 | 19812.809753624886 | - +----------+-----------+--------------------+ + +--------+---------+--------------------+ + | gender | balance | val | + |--------+---------+--------------------| + | F | 32838 | 0.0 | + | M | 4180 | 0.0 | + | M | 5686 | 1064.9028124669405 | + | M | 39225 | 19812.809753624886 | + +--------+---------+--------------------+ VAR_POP ------- @@ -200,14 +200,14 @@ Here is an example for ``SUM`` function:: ... ) AS val ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+--------------------+ - | gender | balance | val | - |----------+-----------+--------------------| - | F | 32838 | 0.0 | - | M | 4180 | 0.0 | - | M | 5686 | 567009.0 | - | M | 39225 | 261698286.88888893 | - +----------+-----------+--------------------+ + +--------+---------+--------------------+ + | gender | balance | val | + |--------+---------+--------------------| + | F | 32838 | 0.0 | + | M | 4180 | 0.0 | + | M | 5686 | 567009.0 | + | M | 39225 | 261698286.88888893 | + +--------+---------+--------------------+ VAR_SAMP -------- @@ -221,14 +221,14 @@ Here is an example for ``SUM`` function:: ... ) AS val ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------------------+ - | gender | balance | val | - |----------+-----------+-------------------| - | F | 32838 | 0.0 | - | M | 4180 | 0.0 | - | M | 5686 | 1134018.0 | - | M | 39225 | 392547430.3333334 | - +----------+-----------+-------------------+ + +--------+---------+-------------------+ + | gender | balance | val | + |--------+---------+-------------------| + | F | 32838 | 0.0 | + | M | 4180 | 0.0 | + | M | 5686 | 1134018.0 | + | M | 39225 | 392547430.3333334 | + +--------+---------+-------------------+ Ranking Functions @@ -248,14 +248,14 @@ ROW_NUMBER os> SELECT gender, balance, ROW_NUMBER() OVER(PARTITION BY gender ORDER BY balance) AS num FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | num | - |----------+-----------+-------| - | F | 32838 | 1 | - | M | 4180 | 1 | - | M | 5686 | 2 | - | M | 39225 | 3 | - +----------+-----------+-------+ + +--------+---------+-----+ + | gender | balance | num | + |--------+---------+-----| + | F | 32838 | 1 | + | M | 4180 | 1 | + | M | 5686 | 2 | + | M | 39225 | 3 | + +--------+---------+-----+ Similarly as regular ``ORDER BY`` clause, you can specify null ordering by ``NULLS FIRST`` or ``NULLS LAST`` which has exactly same behavior:: @@ -267,14 +267,14 @@ Similarly as regular ``ORDER BY`` clause, you can specify null ordering by ``NUL ... FROM accounts ... ORDER BY employer NULLS LAST; fetched rows / total rows = 4/4 - +------------+-------+ - | employer | num | - |------------+-------| - | Netagy | 1 | - | Pyrami | 2 | - | Quility | 3 | - | null | 4 | - +------------+-------+ + +----------+-----+ + | employer | num | + |----------+-----| + | Netagy | 1 | + | Pyrami | 2 | + | Quility | 3 | + | null | 4 | + +----------+-----+ RANK ---- @@ -283,14 +283,14 @@ RANK os> SELECT gender, RANK() OVER(ORDER BY gender DESC) AS rnk FROM accounts; fetched rows / total rows = 4/4 - +----------+-------+ - | gender | rnk | - |----------+-------| - | M | 1 | - | M | 1 | - | M | 1 | - | F | 4 | - +----------+-------+ + +--------+-----+ + | gender | rnk | + |--------+-----| + | M | 1 | + | M | 1 | + | M | 1 | + | F | 4 | + +--------+-----+ DENSE_RANK @@ -300,12 +300,12 @@ Similarly as ``RANK``, ``DENSE_RANK`` function also assigns a rank to each row. os> SELECT gender, DENSE_RANK() OVER(ORDER BY gender DESC) AS rnk FROM accounts; fetched rows / total rows = 4/4 - +----------+-------+ - | gender | rnk | - |----------+-------| - | M | 1 | - | M | 1 | - | M | 1 | - | F | 2 | - +----------+-------+ + +--------+-----+ + | gender | rnk | + |--------+-----| + | M | 1 | + | M | 1 | + | M | 1 | + | F | 2 | + +--------+-----+ diff --git a/docs/user/general/comments.rst b/docs/user/general/comments.rst index ab959da342..536843695e 100644 --- a/docs/user/general/comments.rst +++ b/docs/user/general/comments.rst @@ -26,11 +26,11 @@ A single-line comment starts with either ``#`` or ``--``. All characters in the ... -- comments ... 123; -- comments fetched rows / total rows = 1/1 - +-------+ - | 123 | - |-------| - | 123 | - +-------+ + +-----+ + | 123 | + |-----| + | 123 | + +-----+ Note that double-dash style requires at least one whitespace followed. @@ -48,10 +48,10 @@ A block comment is enclosed within ``/*`` and ``*/`` across one or multiple line ... /* comments */ ... 123; fetched rows / total rows = 1/1 - +-------+ - | 123 | - |-------| - | 123 | - +-------+ + +-----+ + | 123 | + |-----| + | 123 | + +-----+ Additionally, ``/*! ... */`` is supported though ignored for now. This may be used to support optimization hints in future. diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index b4cb69c696..a6cbe02115 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -192,11 +192,11 @@ Here are a few examples for implicit type conversion:: ... 'True' = true, ... DATE('2021-06-10') < '2021-06-11'; fetched rows / total rows = 1/1 - +-----------+-----------------+-------------------------------------+ - | 1 = 1.0 | 'True' = true | DATE('2021-06-10') < '2021-06-11' | - |-----------+-----------------+-------------------------------------| - | True | True | True | - +-----------+-----------------+-------------------------------------+ + +---------+---------------+-----------------------------------+ + | 1 = 1.0 | 'True' = true | DATE('2021-06-10') < '2021-06-11' | + |---------+---------------+-----------------------------------| + | True | True | True | + +---------+---------------+-----------------------------------+ Here are a few examples for explicit type conversion:: @@ -205,11 +205,11 @@ Here are a few examples for explicit type conversion:: ... CAST(1.2 AS STRING), ... CAST('2021-06-10 00:00:00' AS TIMESTAMP); fetched rows / total rows = 1/1 - +---------------------+-----------------------+--------------------------------------------+ - | CAST(true AS INT) | CAST(1.2 AS STRING) | CAST('2021-06-10 00:00:00' AS TIMESTAMP) | - |---------------------+-----------------------+--------------------------------------------| - | 1 | 1.2 | 2021-06-10 00:00:00 | - +---------------------+-----------------------+--------------------------------------------+ + +-------------------+---------------------+------------------------------------------+ + | CAST(true AS INT) | CAST(1.2 AS STRING) | CAST('2021-06-10 00:00:00' AS TIMESTAMP) | + |-------------------+---------------------+------------------------------------------| + | 1 | 1.2 | 2021-06-10 00:00:00 | + +-------------------+---------------------+------------------------------------------+ Undefined Data Type =================== @@ -220,11 +220,11 @@ Here are examples for NULL literal and expressions with NULL literal involved:: os> SELECT NULL, NULL = NULL, 1 + NULL, LENGTH(NULL); fetched rows / total rows = 1/1 - +--------+---------------+------------+----------------+ - | NULL | NULL = NULL | 1 + NULL | LENGTH(NULL) | - |--------+---------------+------------+----------------| - | null | null | null | null | - +--------+---------------+------------+----------------+ + +------+-------------+----------+--------------+ + | NULL | NULL = NULL | 1 + NULL | LENGTH(NULL) | + |------+-------------+----------+--------------| + | null | null | null | null | + +------+-------------+----------+--------------+ Numeric Data Types @@ -347,11 +347,11 @@ A string can also represent and be converted to date and time types (except to i ... '2021-06-18' < DATE('2021-06-17'), ... '10:20:00' <= TIME('11:00:00'); fetched rows / total rows = 1/1 - +------------------------------------------------------------+-------------------------------------+----------------------------------+ - | TIMESTAMP('2021-06-17 00:00:00') = '2021-06-17 00:00:00' | '2021-06-18' < DATE('2021-06-17') | '10:20:00' <= TIME('11:00:00') | - |------------------------------------------------------------+-------------------------------------+----------------------------------| - | True | False | True | - +------------------------------------------------------------+-------------------------------------+----------------------------------+ + +----------------------------------------------------------+-----------------------------------+--------------------------------+ + | TIMESTAMP('2021-06-17 00:00:00') = '2021-06-17 00:00:00' | '2021-06-18' < DATE('2021-06-17') | '10:20:00' <= TIME('11:00:00') | + |----------------------------------------------------------+-----------------------------------+--------------------------------| + | True | False | True | + +----------------------------------------------------------+-----------------------------------+--------------------------------+ Please, see `more examples here <../dql/expressions.rst#toc-entry-15>`_. @@ -478,11 +478,11 @@ A string is a sequence of characters enclosed in either single or double quotes. os> SELECT 'hello', "world", '"hello"', "'world'", '''hello''', """world""" fetched rows / total rows = 1/1 - +-----------+-----------+-------------+-------------+---------------+---------------+ - | 'hello' | "world" | '"hello"' | "'world'" | '''hello''' | """world""" | - |-----------+-----------+-------------+-------------+---------------+---------------| - | hello | world | "hello" | 'world' | 'hello' | "world" | - +-----------+-----------+-------------+-------------+---------------+---------------+ + +---------+---------+-----------+-----------+-------------+-------------+ + | 'hello' | "world" | '"hello"' | "'world'" | '''hello''' | """world""" | + |---------+---------+-----------+-----------+-------------+-------------| + | hello | world | "hello" | 'world' | 'hello' | "world" | + +---------+---------+-----------+-----------+-------------+-------------+ Boolean Data Types ================== @@ -493,8 +493,8 @@ A boolean can be represented by constant value ``TRUE`` or ``FALSE``. Besides, c ... true, FALSE, ... CAST('TRUE' AS boolean), CAST('false' AS boolean); fetched rows / total rows = 1/1 - +--------+---------+---------------------------+----------------------------+ - | true | FALSE | CAST('TRUE' AS boolean) | CAST('false' AS boolean) | - |--------+---------+---------------------------+----------------------------| - | True | False | True | False | - +--------+---------+---------------------------+----------------------------+ + +------+-------+-------------------------+--------------------------+ + | true | FALSE | CAST('TRUE' AS boolean) | CAST('false' AS boolean) | + |------+-------+-------------------------+--------------------------| + | True | False | True | False | + +------+-------+-------------------------+--------------------------+ diff --git a/docs/user/general/identifiers.rst b/docs/user/general/identifiers.rst index fad2fa4b23..562bf38526 100644 --- a/docs/user/general/identifiers.rst +++ b/docs/user/general/identifiers.rst @@ -40,14 +40,14 @@ Here are examples for using index pattern directly without quotes:: os> SELECT * FROM *cc*nts; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ Delimited Identifiers @@ -76,14 +76,14 @@ Here are examples for quoting an index name by back ticks:: os> SELECT * FROM `accounts`; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ Case Sensitivity @@ -121,23 +121,23 @@ The first example is to show a column name qualified by full table name original os> SELECT city, accounts.age, ABS(accounts.balance) FROM accounts WHERE accounts.age < 30; fetched rows / total rows = 1/1 - +--------+-------+-------------------------+ - | city | age | ABS(accounts.balance) | - |--------+-------+-------------------------| - | Nogal | 28 | 32838 | - +--------+-------+-------------------------+ + +-------+-----+-----------------------+ + | city | age | ABS(accounts.balance) | + |-------+-----+-----------------------| + | Nogal | 28 | 32838 | + +-------+-----+-----------------------+ The second example is to show a field name qualified by index alias specified. Similarly, the alias qualifier is optional in this case:: os> SELECT city, acc.age, ABS(acc.balance) FROM accounts AS acc WHERE acc.age > 30; fetched rows / total rows = 3/3 - +--------+-------+--------------------+ - | city | age | ABS(acc.balance) | - |--------+-------+--------------------| - | Brogan | 32 | 39225 | - | Dante | 36 | 5686 | - | Orick | 33 | 4180 | - +--------+-------+--------------------+ + +--------+-----+------------------+ + | city | age | ABS(acc.balance) | + |--------+-----+------------------| + | Brogan | 32 | 39225 | + | Dante | 36 | 5686 | + | Orick | 33 | 4180 | + +--------+-----+------------------+ Note that in both examples above, the qualifier is removed in response. This happens only when identifiers selected is a simple field name. In other cases, expressions rather than an atom field, the column name in response is exactly the same as the text in ``SELECT``clause. @@ -160,22 +160,22 @@ Query wildcard indices:: os> SELECT count(*) as cnt FROM acc*; fetched rows / total rows = 1/1 - +-------+ - | cnt | - |-------| - | 5 | - +-------+ + +-----+ + | cnt | + |-----| + | 5 | + +-----+ Query delimited multiple indices seperated by ``,``:: os> SELECT count(*) as cnt FROM `accounts,account2`; fetched rows / total rows = 1/1 - +-------+ - | cnt | - |-------| - | 5 | - +-------+ + +-----+ + | cnt | + |-----| + | 5 | + +-----+ diff --git a/docs/user/general/values.rst b/docs/user/general/values.rst index 178609f175..f675b42e75 100644 --- a/docs/user/general/values.rst +++ b/docs/user/general/values.rst @@ -19,14 +19,14 @@ Here is an example, Nanette doesn't have email field and Dail has employer filed os> SELECT firstname, employer, email FROM accounts; fetched rows / total rows = 4/4 - +-------------+------------+-----------------------+ - | firstname | employer | email | - |-------------+------------+-----------------------| - | Amber | Pyrami | amberduke@pyrami.com | - | Hattie | Netagy | hattiebond@netagy.com | - | Nanette | Quility | null | - | Dale | null | daleadams@boink.com | - +-------------+------------+-----------------------+ + +-----------+----------+-----------------------+ + | firstname | employer | email | + |-----------+----------+-----------------------| + | Amber | Pyrami | amberduke@pyrami.com | + | Hattie | Netagy | hattiebond@netagy.com | + | Nanette | Quility | null | + | Dale | null | daleadams@boink.com | + +-----------+----------+-----------------------+ General NULL and MISSING Values Handling @@ -37,14 +37,14 @@ Here is an example:: os> SELECT firstname, employer LIKE 'Quility', email LIKE '%com' FROM accounts; fetched rows / total rows = 4/4 - +-------------+---------------------------+---------------------+ - | firstname | employer LIKE 'Quility' | email LIKE '%com' | - |-------------+---------------------------+---------------------| - | Amber | False | True | - | Hattie | False | True | - | Nanette | True | null | - | Dale | null | True | - +-------------+---------------------------+---------------------+ + +-----------+-------------------------+-------------------+ + | firstname | employer LIKE 'Quility' | email LIKE '%com' | + |-----------+-------------------------+-------------------| + | Amber | False | True | + | Hattie | False | True | + | Nanette | True | null | + | Dale | null | True | + +-----------+-------------------------+-------------------+ Special NULL and MISSING Values Handling ---------------------------------------- diff --git a/docs/user/ppl/admin/connectors/prometheus_connector.rst b/docs/user/ppl/admin/connectors/prometheus_connector.rst index 1dfe6cda22..812df4f894 100644 --- a/docs/user/ppl/admin/connectors/prometheus_connector.rst +++ b/docs/user/ppl/admin/connectors/prometheus_connector.rst @@ -87,16 +87,16 @@ Sample Example:: > source = my_prometheus.prometheus_http_requests_total; - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ - | @value | @timestamp | handler | code | instance | job | - |------------+------------------------+--------------------------------+---------------+-------------+-------------| - | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | - | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | - | 11 | "2022-11-03 07:18:64" |"/-/metrics" | 500 | 192.15.2.1 | prometheus | - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +--------+-----------------------+--------------+------+------------+------------+ + | @value | @timestamp | handler | code | instance | job | + |--------+-----------------------+--------------+------+------------+------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" | "/-/metrics" | 500 | 192.15.2.1 | prometheus | + +--------+-----------------------+--------------+------+------------+------------+ @@ -119,30 +119,30 @@ Example queries 1. Metric Selection Query:: > source = my_prometheus.prometheus_http_requests_total - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ - | @value | @timestamp | handler | code | instance | job | - |------------+------------------------+--------------------------------+---------------+-------------+-------------| - | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | - | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | - | 11 | "2022-11-03 07:18:64" |"/-/metrics" | 500 | 192.15.2.1 | prometheus | - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +--------+-----------------------+--------------+------+------------+------------+ + | @value | @timestamp | handler | code | instance | job | + |--------+-----------------------+--------------+------+------------+------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" | "/-/metrics" | 500 | 192.15.2.1 | prometheus | + +--------+-----------------------+--------------+------+------------+------------+ 2. Metric Selecting Query with specific dimensions:: > source = my_prometheus.prometheus_http_requests_total | where handler='/-/ready' and code='200' - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ - | @value | @timestamp | handler | code | instance | job | - |------------+------------------------+--------------------------------+---------------+-------------+-------------| - | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 2 | "2022-11-03 07:18:44" | "/-/ready" | 200 | 192.15.2.1 | prometheus | - | 9 | "2022-11-03 07:18:54" | "/-/ready" | 200 | 192.15.2.1 | prometheus | - | 11 | "2022-11-03 07:18:64" | "/-/ready" | 200 | 192.15.2.1 | prometheus | - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +--------+-----------------------+------------+------+------------+------------+ + | @value | @timestamp | handler | code | instance | job | + |--------+-----------------------+------------+------+------------+------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 200 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/ready" | 200 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" | "/-/ready" | 200 | 192.15.2.1 | prometheus | + +--------+-----------------------+------------+------+------------+------------+ 3. Average aggregation on a metric:: @@ -199,16 +199,16 @@ PromQL Support for prometheus Connector Example:: > source=my_prometheus.query_range('prometheus_http_requests_total', 1686694425, 1686700130, 14) - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ - | @value | @timestamp | handler | code | instance | job | - |------------+------------------------+--------------------------------+---------------+-------------+-------------| - | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | - | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | - | 11 | "2022-11-03 07:18:64" |"/-/metrics" | 500 | 192.15.2.1 | prometheus | - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +--------+-----------------------+--------------+------+------------+------------+ + | @value | @timestamp | handler | code | instance | job | + |--------+-----------------------+--------------+------+------------+------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" | "/-/metrics" | 500 | 192.15.2.1 | prometheus | + +--------+-----------------------+--------------+------+------------+------------+ Prometheus Connector Table Functions diff --git a/docs/user/ppl/admin/cross_cluster_search.rst b/docs/user/ppl/admin/cross_cluster_search.rst index f57ea288e8..4b267a9340 100644 --- a/docs/user/ppl/admin/cross_cluster_search.rst +++ b/docs/user/ppl/admin/cross_cluster_search.rst @@ -40,14 +40,14 @@ Example PPL query:: os> source=my_remote_cluster:accounts; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ Limitation diff --git a/docs/user/ppl/cmd/ad.rst b/docs/user/ppl/cmd/ad.rst index 103c7f7483..5d7a572c96 100644 --- a/docs/user/ppl/cmd/ad.rst +++ b/docs/user/ppl/cmd/ad.rst @@ -50,11 +50,11 @@ PPL query:: > source=nyc_taxi | fields value, timestamp | AD time_field='timestamp' | where value=10844.0 fetched rows / total rows = 1/1 - +---------+---------------------+---------+-----------------+ - | value | timestamp | score | anomaly_grade | - |---------+---------------------+---------+-----------------| - | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | - +---------+---------------------+---------+-----------------+ + +---------+---------------------+-------+---------------+ + | value | timestamp | score | anomaly_grade | + |---------+---------------------+-------+---------------| + | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + +---------+---------------------+-------+---------------+ Example 2: Detecting events in New York City from taxi ridership data with time-series data independently with each category ============================================================================================================================ @@ -65,12 +65,12 @@ PPL query:: > source=nyc_taxi | fields category, value, timestamp | AD time_field='timestamp' category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 - +------------+---------+---------------------+---------+-----------------+ - | category | value | timestamp | score | anomaly_grade | - |------------+---------+---------------------+---------+-----------------| - | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | - | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | - +------------+---------+---------------------+---------+-----------------+ + +----------+---------+---------------------+-------+---------------+ + | category | value | timestamp | score | anomaly_grade | + |----------+---------+---------------------+-------+---------------| + | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | + +----------+---------+---------------------+-------+---------------+ Example 3: Detecting events in New York City from taxi ridership data with non-time-series data @@ -82,11 +82,11 @@ PPL query:: > source=nyc_taxi | fields value | AD | where value=10844.0 fetched rows / total rows = 1/1 - +---------+---------+-------------+ - | value | score | anomalous | - |---------+---------+-------------| - | 10844.0 | 0.0 | False | - +---------+---------+-------------+ + +---------+-------+-----------+ + | value | score | anomalous | + |---------+-------+-----------| + | 10844.0 | 0.0 | False | + +---------+-------+-----------+ Example 4: Detecting events in New York City from taxi ridership data with non-time-series data independently with each category ================================================================================================================================ @@ -97,9 +97,9 @@ PPL query:: > source=nyc_taxi | fields category, value | AD category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 - +------------+---------+---------+-------------+ - | category | value | score | anomalous | - |------------+---------+---------+-------------| - | night | 10844.0 | 0.0 | False | - | day | 6526.0 | 0.0 | False | - +------------+---------+---------+-------------+ + +----------+---------+-------+-----------+ + | category | value | score | anomalous | + |----------+---------+-------+-----------| + | night | 10844.0 | 0.0 | False | + | day | 6526.0 | 0.0 | False | + +----------+---------+-------+-----------+ diff --git a/docs/user/ppl/cmd/dedup.rst b/docs/user/ppl/cmd/dedup.rst index ebceb9e0bd..362d1637f7 100644 --- a/docs/user/ppl/cmd/dedup.rst +++ b/docs/user/ppl/cmd/dedup.rst @@ -34,12 +34,12 @@ PPL query:: os> source=accounts | dedup gender | fields account_number, gender; fetched rows / total rows = 2/2 - +------------------+----------+ - | account_number | gender | - |------------------+----------| - | 1 | M | - | 13 | F | - +------------------+----------+ + +----------------+--------+ + | account_number | gender | + |----------------+--------| + | 1 | M | + | 13 | F | + +----------------+--------+ Example 2: Keep 2 duplicates documents ====================================== @@ -50,13 +50,13 @@ PPL query:: os> source=accounts | dedup 2 gender | fields account_number, gender; fetched rows / total rows = 3/3 - +------------------+----------+ - | account_number | gender | - |------------------+----------| - | 1 | M | - | 6 | M | - | 13 | F | - +------------------+----------+ + +----------------+--------+ + | account_number | gender | + |----------------+--------| + | 1 | M | + | 6 | M | + | 13 | F | + +----------------+--------+ Example 3: Keep or Ignore the empty field by default ============================================ @@ -67,14 +67,14 @@ PPL query:: os> source=accounts | dedup email keepempty=true | fields account_number, email; fetched rows / total rows = 4/4 - +------------------+-----------------------+ - | account_number | email | - |------------------+-----------------------| - | 1 | amberduke@pyrami.com | - | 6 | hattiebond@netagy.com | - | 13 | null | - | 18 | daleadams@boink.com | - +------------------+-----------------------+ + +----------------+-----------------------+ + | account_number | email | + |----------------+-----------------------| + | 1 | amberduke@pyrami.com | + | 6 | hattiebond@netagy.com | + | 13 | null | + | 18 | daleadams@boink.com | + +----------------+-----------------------+ The example show dedup the document by ignore the empty value field. @@ -83,13 +83,13 @@ PPL query:: os> source=accounts | dedup email | fields account_number, email; fetched rows / total rows = 3/3 - +------------------+-----------------------+ - | account_number | email | - |------------------+-----------------------| - | 1 | amberduke@pyrami.com | - | 6 | hattiebond@netagy.com | - | 18 | daleadams@boink.com | - +------------------+-----------------------+ + +----------------+-----------------------+ + | account_number | email | + |----------------+-----------------------| + | 1 | amberduke@pyrami.com | + | 6 | hattiebond@netagy.com | + | 18 | daleadams@boink.com | + +----------------+-----------------------+ Example 4: Dedup in consecutive document @@ -101,13 +101,13 @@ PPL query:: os> source=accounts | dedup gender consecutive=true | fields account_number, gender; fetched rows / total rows = 3/3 - +------------------+----------+ - | account_number | gender | - |------------------+----------| - | 1 | M | - | 13 | F | - | 18 | M | - +------------------+----------+ + +----------------+--------+ + | account_number | gender | + |----------------+--------| + | 1 | M | + | 13 | F | + | 18 | M | + +----------------+--------+ Limitation ========== diff --git a/docs/user/ppl/cmd/describe.rst b/docs/user/ppl/cmd/describe.rst index a0ecbd3169..2b03ceda57 100644 --- a/docs/user/ppl/cmd/describe.rst +++ b/docs/user/ppl/cmd/describe.rst @@ -33,21 +33,21 @@ PPL query:: os> describe accounts; fetched rows / total rows = 11/11 - +----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ - | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | - |----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------| - | docTestCluster | null | accounts | account_number | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 0 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | address | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 2 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | balance | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 3 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | gender | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 4 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | city | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 5 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | employer | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 6 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | state | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 7 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | age | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 8 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | email | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 9 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | - +----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ + +----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | + |----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------| + | docTestCluster | null | accounts | account_number | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 0 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | address | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 2 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | balance | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 3 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | gender | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 4 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | city | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 5 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | employer | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 6 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | state | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 7 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | age | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 8 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | email | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 9 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | + +----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ Example 2: Fetch metadata with condition and filter =================================================== @@ -76,13 +76,13 @@ PPL query:: os> describe my_prometheus.prometheus_http_requests_total; fetched rows / total rows = 6/6 - +-----------------+----------------+--------------------------------+---------------+-------------+ - | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | COLUMN_NAME | DATA_TYPE | - |-----------------+----------------+--------------------------------+---------------+-------------| - | my_prometheus | default | prometheus_http_requests_total | handler | keyword | - | my_prometheus | default | prometheus_http_requests_total | code | keyword | - | my_prometheus | default | prometheus_http_requests_total | instance | keyword | - | my_prometheus | default | prometheus_http_requests_total | @timestamp | timestamp | - | my_prometheus | default | prometheus_http_requests_total | @value | double | - | my_prometheus | default | prometheus_http_requests_total | job | keyword | - +-----------------+----------------+--------------------------------+---------------+-------------+ + +---------------+--------------+--------------------------------+-------------+-----------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | COLUMN_NAME | DATA_TYPE | + |---------------+--------------+--------------------------------+-------------+-----------| + | my_prometheus | default | prometheus_http_requests_total | handler | keyword | + | my_prometheus | default | prometheus_http_requests_total | code | keyword | + | my_prometheus | default | prometheus_http_requests_total | instance | keyword | + | my_prometheus | default | prometheus_http_requests_total | @timestamp | timestamp | + | my_prometheus | default | prometheus_http_requests_total | @value | double | + | my_prometheus | default | prometheus_http_requests_total | job | keyword | + +---------------+--------------+--------------------------------+-------------+-----------+ diff --git a/docs/user/ppl/cmd/eval.rst b/docs/user/ppl/cmd/eval.rst index 48a14ae0a8..c950028674 100644 --- a/docs/user/ppl/cmd/eval.rst +++ b/docs/user/ppl/cmd/eval.rst @@ -30,14 +30,14 @@ PPL query:: os> source=accounts | eval doubleAge = age * 2 | fields age, doubleAge ; fetched rows / total rows = 4/4 - +-------+-------------+ - | age | doubleAge | - |-------+-------------| - | 32 | 64 | - | 36 | 72 | - | 28 | 56 | - | 33 | 66 | - +-------+-------------+ + +-----+-----------+ + | age | doubleAge | + |-----+-----------| + | 32 | 64 | + | 36 | 72 | + | 28 | 56 | + | 33 | 66 | + +-----+-----------+ Example 2: Override the existing field @@ -49,14 +49,14 @@ PPL query:: os> source=accounts | eval age = age + 1 | fields age ; fetched rows / total rows = 4/4 - +-------+ - | age | - |-------| - | 33 | - | 37 | - | 29 | - | 34 | - +-------+ + +-----+ + | age | + |-----| + | 33 | + | 37 | + | 29 | + | 34 | + +-----+ Example 3: Create the new field with field defined in eval ========================================================== @@ -67,14 +67,14 @@ PPL query:: os> source=accounts | eval doubleAge = age * 2, ddAge = doubleAge * 2 | fields age, doubleAge, ddAge ; fetched rows / total rows = 4/4 - +-------+-------------+---------+ - | age | doubleAge | ddAge | - |-------+-------------+---------| - | 32 | 64 | 128 | - | 36 | 72 | 144 | - | 28 | 56 | 112 | - | 33 | 66 | 132 | - +-------+-------------+---------+ + +-----+-----------+-------+ + | age | doubleAge | ddAge | + |-----+-----------+-------| + | 32 | 64 | 128 | + | 36 | 72 | 144 | + | 28 | 56 | 112 | + | 33 | 66 | 132 | + +-----+-----------+-------+ Limitation ========== diff --git a/docs/user/ppl/cmd/fields.rst b/docs/user/ppl/cmd/fields.rst index dbae5b20a4..32c3a665d7 100644 --- a/docs/user/ppl/cmd/fields.rst +++ b/docs/user/ppl/cmd/fields.rst @@ -31,14 +31,14 @@ PPL query:: os> source=accounts | fields account_number, firstname, lastname; fetched rows / total rows = 4/4 - +------------------+-------------+------------+ - | account_number | firstname | lastname | - |------------------+-------------+------------| - | 1 | Amber | Duke | - | 6 | Hattie | Bond | - | 13 | Nanette | Bates | - | 18 | Dale | Adams | - +------------------+-------------+------------+ + +----------------+-----------+----------+ + | account_number | firstname | lastname | + |----------------+-----------+----------| + | 1 | Amber | Duke | + | 6 | Hattie | Bond | + | 13 | Nanette | Bates | + | 18 | Dale | Adams | + +----------------+-----------+----------+ Example 2: Remove specified fields from result ============================================== @@ -49,12 +49,12 @@ PPL query:: os> source=accounts | fields account_number, firstname, lastname | fields - account_number ; fetched rows / total rows = 4/4 - +-------------+------------+ - | firstname | lastname | - |-------------+------------| - | Amber | Duke | - | Hattie | Bond | - | Nanette | Bates | - | Dale | Adams | - +-------------+------------+ + +-----------+----------+ + | firstname | lastname | + |-----------+----------| + | Amber | Duke | + | Hattie | Bond | + | Nanette | Bates | + | Dale | Adams | + +-----------+----------+ diff --git a/docs/user/ppl/cmd/grok.rst b/docs/user/ppl/cmd/grok.rst index 6a121c7431..35f3b0c846 100644 --- a/docs/user/ppl/cmd/grok.rst +++ b/docs/user/ppl/cmd/grok.rst @@ -72,14 +72,14 @@ PPL query:: os> source=apache | grok message '%{COMMONAPACHELOG}' | fields COMMONAPACHELOG, timestamp, response, bytes ; fetched rows / total rows = 4/4 - +-----------------------------------------------------------------------------------------------------------------------------+----------------------------+------------+---------+ - | COMMONAPACHELOG | timestamp | response | bytes | - |-----------------------------------------------------------------------------------------------------------------------------+----------------------------+------------+---------| - | 177.95.8.74 - upton5450 [28/Sep/2022:10:15:57 -0700] "HEAD /e-business/mindshare HTTP/1.0" 404 19927 | 28/Sep/2022:10:15:57 -0700 | 404 | 19927 | - | 127.45.152.6 - pouros8756 [28/Sep/2022:10:15:57 -0700] "GET /architectures/convergence/niches/mindshare HTTP/1.0" 100 28722 | 28/Sep/2022:10:15:57 -0700 | 100 | 28722 | - | 118.223.210.105 - - [28/Sep/2022:10:15:57 -0700] "PATCH /strategize/out-of-the-box HTTP/1.0" 401 27439 | 28/Sep/2022:10:15:57 -0700 | 401 | 27439 | - | 210.204.15.104 - - [28/Sep/2022:10:15:57 -0700] "POST /users HTTP/1.1" 301 9481 | 28/Sep/2022:10:15:57 -0700 | 301 | 9481 | - +-----------------------------------------------------------------------------------------------------------------------------+----------------------------+------------+---------+ + +-----------------------------------------------------------------------------------------------------------------------------+----------------------------+----------+-------+ + | COMMONAPACHELOG | timestamp | response | bytes | + |-----------------------------------------------------------------------------------------------------------------------------+----------------------------+----------+-------| + | 177.95.8.74 - upton5450 [28/Sep/2022:10:15:57 -0700] "HEAD /e-business/mindshare HTTP/1.0" 404 19927 | 28/Sep/2022:10:15:57 -0700 | 404 | 19927 | + | 127.45.152.6 - pouros8756 [28/Sep/2022:10:15:57 -0700] "GET /architectures/convergence/niches/mindshare HTTP/1.0" 100 28722 | 28/Sep/2022:10:15:57 -0700 | 100 | 28722 | + | 118.223.210.105 - - [28/Sep/2022:10:15:57 -0700] "PATCH /strategize/out-of-the-box HTTP/1.0" 401 27439 | 28/Sep/2022:10:15:57 -0700 | 401 | 27439 | + | 210.204.15.104 - - [28/Sep/2022:10:15:57 -0700] "POST /users HTTP/1.1" 301 9481 | 28/Sep/2022:10:15:57 -0700 | 301 | 9481 | + +-----------------------------------------------------------------------------------------------------------------------------+----------------------------+----------+-------+ Limitations =========== diff --git a/docs/user/ppl/cmd/head.rst b/docs/user/ppl/cmd/head.rst index 1b4599f5de..cd4aed5a54 100644 --- a/docs/user/ppl/cmd/head.rst +++ b/docs/user/ppl/cmd/head.rst @@ -30,14 +30,14 @@ PPL query:: os> source=accounts | fields firstname, age | head; fetched rows / total rows = 4/4 - +-------------+-------+ - | firstname | age | - |-------------+-------| - | Amber | 32 | - | Hattie | 36 | - | Nanette | 28 | - | Dale | 33 | - +-------------+-------+ + +-----------+-----+ + | firstname | age | + |-----------+-----| + | Amber | 32 | + | Hattie | 36 | + | Nanette | 28 | + | Dale | 33 | + +-----------+-----+ Example 2: Get first N results =========================================== @@ -48,13 +48,13 @@ PPL query:: os> source=accounts | fields firstname, age | head 3; fetched rows / total rows = 3/3 - +-------------+-------+ - | firstname | age | - |-------------+-------| - | Amber | 32 | - | Hattie | 36 | - | Nanette | 28 | - +-------------+-------+ + +-----------+-----+ + | firstname | age | + |-----------+-----| + | Amber | 32 | + | Hattie | 36 | + | Nanette | 28 | + +-----------+-----+ Example 3: Get first N results after offset M ============================================= @@ -65,13 +65,13 @@ PPL query:: os> source=accounts | fields firstname, age | head 3 from 1; fetched rows / total rows = 3/3 - +-------------+-------+ - | firstname | age | - |-------------+-------| - | Hattie | 36 | - | Nanette | 28 | - | Dale | 33 | - +-------------+-------+ + +-----------+-----+ + | firstname | age | + |-----------+-----| + | Hattie | 36 | + | Nanette | 28 | + | Dale | 33 | + +-----------+-----+ Limitation ========== diff --git a/docs/user/ppl/cmd/information_schema.rst b/docs/user/ppl/cmd/information_schema.rst index 26341d6972..4210502eda 100644 --- a/docs/user/ppl/cmd/information_schema.rst +++ b/docs/user/ppl/cmd/information_schema.rst @@ -29,11 +29,11 @@ PPL query for fetching PROMETHEUS TABLES with where clause:: os> source = my_prometheus.information_schema.tables | where TABLE_NAME='prometheus_http_requests_total' fetched rows / total rows = 1/1 - +-----------------+----------------+--------------------------------+--------------+--------+---------------------------+ - | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | - |-----------------+----------------+--------------------------------+--------------+--------+---------------------------| - | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | - +-----------------+----------------+--------------------------------+--------------+--------+---------------------------+ + +---------------+--------------+--------------------------------+------------+------+---------------------------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | + |---------------+--------------+--------------------------------+------------+------+---------------------------| + | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | + +---------------+--------------+--------------------------------+------------+------+---------------------------+ Example 2: Search tables in prometheus datasource. @@ -45,13 +45,13 @@ PPL query for searching PROMETHEUS TABLES:: os> source = my_prometheus.information_schema.tables | where LIKE(TABLE_NAME, "%http%"); fetched rows / total rows = 6/6 - +-----------------+----------------+--------------------------------------------+--------------+--------+----------------------------------------------------+ - | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | - |-----------------+----------------+--------------------------------------------+--------------+--------+----------------------------------------------------| - | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | - | my_prometheus | default | promhttp_metric_handler_requests_in_flight | gauge | | Current number of scrapes being served. | - | my_prometheus | default | prometheus_http_request_duration_seconds | histogram | | Histogram of latencies for HTTP requests. | - | my_prometheus | default | prometheus_sd_http_failures_total | counter | | Number of HTTP service discovery refresh failures. | - | my_prometheus | default | promhttp_metric_handler_requests_total | counter | | Total number of scrapes by HTTP status code. | - | my_prometheus | default | prometheus_http_response_size_bytes | histogram | | Histogram of response size for HTTP requests. | - +-----------------+----------------+--------------------------------------------+--------------+--------+----------------------------------------------------+ + +---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | + |---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------| + | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | + | my_prometheus | default | promhttp_metric_handler_requests_in_flight | gauge | | Current number of scrapes being served. | + | my_prometheus | default | prometheus_http_request_duration_seconds | histogram | | Histogram of latencies for HTTP requests. | + | my_prometheus | default | prometheus_sd_http_failures_total | counter | | Number of HTTP service discovery refresh failures. | + | my_prometheus | default | promhttp_metric_handler_requests_total | counter | | Total number of scrapes by HTTP status code. | + | my_prometheus | default | prometheus_http_response_size_bytes | histogram | | Histogram of response size for HTTP requests. | + +---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------+ diff --git a/docs/user/ppl/cmd/ml.rst b/docs/user/ppl/cmd/ml.rst index 2e04674c1e..a48c1ec589 100644 --- a/docs/user/ppl/cmd/ml.rst +++ b/docs/user/ppl/cmd/ml.rst @@ -56,11 +56,11 @@ PPL query:: os> source=nyc_taxi | fields value, timestamp | ml action='train' algorithm='rcf' time_field='timestamp' | where value=10844.0 fetched rows / total rows = 1/1 - +---------+---------------------+---------+-----------------+ - | value | timestamp | score | anomaly_grade | - |---------+---------------------+---------+-----------------| - | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | - +---------+---------------------+---------+-----------------+ + +---------+---------------------+-------+---------------+ + | value | timestamp | score | anomaly_grade | + |---------+---------------------+-------+---------------| + | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + +---------+---------------------+-------+---------------+ Example 2: Detecting events in New York City from taxi ridership data with time-series data independently with each category ============================================================================================================================ @@ -71,12 +71,12 @@ PPL query:: os> source=nyc_taxi | fields category, value, timestamp | ml action='train' algorithm='rcf' time_field='timestamp' category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 - +------------+---------+---------------------+---------+-----------------+ - | category | value | timestamp | score | anomaly_grade | - |------------+---------+---------------------+---------+-----------------| - | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | - | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | - +------------+---------+---------------------+---------+-----------------+ + +----------+---------+---------------------+-------+---------------+ + | category | value | timestamp | score | anomaly_grade | + |----------+---------+---------------------+-------+---------------| + | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | + +----------+---------+---------------------+-------+---------------+ Example 3: Detecting events in New York City from taxi ridership data with non-time-series data @@ -88,11 +88,11 @@ PPL query:: os> source=nyc_taxi | fields value | ml action='train' algorithm='rcf' | where value=10844.0 fetched rows / total rows = 1/1 - +---------+---------+-------------+ - | value | score | anomalous | - |---------+---------+-------------| - | 10844.0 | 0.0 | False | - +---------+---------+-------------+ + +---------+-------+-----------+ + | value | score | anomalous | + |---------+-------+-----------| + | 10844.0 | 0.0 | False | + +---------+-------+-----------+ Example 4: Detecting events in New York City from taxi ridership data with non-time-series data independently with each category ================================================================================================================================ @@ -103,12 +103,12 @@ PPL query:: os> source=nyc_taxi | fields category, value | ml action='train' algorithm='rcf' category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 - +------------+---------+---------+-------------+ - | category | value | score | anomalous | - |------------+---------+---------+-------------| - | night | 10844.0 | 0.0 | False | - | day | 6526.0 | 0.0 | False | - +------------+---------+---------+-------------+ + +----------+---------+-------+-----------+ + | category | value | score | anomalous | + |----------+---------+-------+-----------| + | night | 10844.0 | 0.0 | False | + | day | 6526.0 | 0.0 | False | + +----------+---------+-------+-----------+ KMEANS ====== diff --git a/docs/user/ppl/cmd/parse.rst b/docs/user/ppl/cmd/parse.rst index 82eff8ee85..d1015cccb9 100644 --- a/docs/user/ppl/cmd/parse.rst +++ b/docs/user/ppl/cmd/parse.rst @@ -72,13 +72,13 @@ PPL query:: os> source=accounts | parse address '(?\d+) (?.+)' | where cast(streetNumber as int) > 500 | sort num(streetNumber) | fields streetNumber, street ; fetched rows / total rows = 3/3 - +----------------+----------------+ - | streetNumber | street | - |----------------+----------------| - | 671 | Bristol Street | - | 789 | Madison Street | - | 880 | Holmes Lane | - +----------------+----------------+ + +--------------+----------------+ + | streetNumber | street | + |--------------+----------------| + | 671 | Bristol Street | + | 789 | Madison Street | + | 880 | Holmes Lane | + +--------------+----------------+ Limitations =========== diff --git a/docs/user/ppl/cmd/patterns.rst b/docs/user/ppl/cmd/patterns.rst index 370404ecb6..13f08d0aa6 100644 --- a/docs/user/ppl/cmd/patterns.rst +++ b/docs/user/ppl/cmd/patterns.rst @@ -31,14 +31,14 @@ PPL query:: os> source=accounts | patterns email | fields email, patterns_field ; fetched rows / total rows = 4/4 - +-----------------------+------------------+ - | email | patterns_field | - |-----------------------+------------------| - | amberduke@pyrami.com | @. | - | hattiebond@netagy.com | @. | - | null | | - | daleadams@boink.com | @. | - +-----------------------+------------------+ + +-----------------------+----------------+ + | email | patterns_field | + |-----------------------+----------------| + | amberduke@pyrami.com | @. | + | hattiebond@netagy.com | @. | + | null | | + | daleadams@boink.com | @. | + +-----------------------+----------------+ Example 2: Extract log patterns =============================== diff --git a/docs/user/ppl/cmd/rare.rst b/docs/user/ppl/cmd/rare.rst index 35b660daa7..f6013711ae 100644 --- a/docs/user/ppl/cmd/rare.rst +++ b/docs/user/ppl/cmd/rare.rst @@ -32,12 +32,12 @@ PPL query:: os> source=accounts | rare gender; fetched rows / total rows = 2/2 - +----------+ - | gender | - |----------| - | F | - | M | - +----------+ + +--------+ + | gender | + |--------| + | F | + | M | + +--------+ Example 2: Find the least common values organized by gender @@ -49,14 +49,14 @@ PPL query:: os> source=accounts | rare age by gender; fetched rows / total rows = 4/4 - +----------+-------+ - | gender | age | - |----------+-------| - | F | 28 | - | M | 32 | - | M | 33 | - | M | 36 | - +----------+-------+ + +--------+-----+ + | gender | age | + |--------+-----| + | F | 28 | + | M | 32 | + | M | 33 | + | M | 36 | + +--------+-----+ Limitation ========== diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index a4383a9f5f..c942884248 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -31,14 +31,14 @@ PPL query:: os> source=accounts | rename account_number as an | fields an; fetched rows / total rows = 4/4 - +------+ - | an | - |------| - | 1 | - | 6 | - | 13 | - | 18 | - +------+ + +----+ + | an | + |----| + | 1 | + | 6 | + | 13 | + | 18 | + +----+ Example 2: Rename multiple fields @@ -50,14 +50,14 @@ PPL query:: os> source=accounts | rename account_number as an, employer as emp | fields an, emp; fetched rows / total rows = 4/4 - +------+---------+ - | an | emp | - |------+---------| - | 1 | Pyrami | - | 6 | Netagy | - | 13 | Quility | - | 18 | null | - +------+---------+ + +----+---------+ + | an | emp | + |----+---------| + | 1 | Pyrami | + | 6 | Netagy | + | 13 | Quility | + | 18 | null | + +----+---------+ Limitation ========== diff --git a/docs/user/ppl/cmd/search.rst b/docs/user/ppl/cmd/search.rst index 5299f9f78a..9e55daddeb 100644 --- a/docs/user/ppl/cmd/search.rst +++ b/docs/user/ppl/cmd/search.rst @@ -37,14 +37,14 @@ PPL query:: os> source=accounts; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ Example 2: Fetch data with condition ==================================== @@ -55,10 +55,10 @@ PPL query:: os> source=accounts account_number=1 or gender="F"; fetched rows / total rows = 2/2 - +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ + +----------------+-----------+--------------------+---------+--------+--------+----------+-------+-----+----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+--------------------+---------+--------+--------+----------+-------+-----+----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + +----------------+-----------+--------------------+---------+--------+--------+----------+-------+-----+----------------------+----------+ diff --git a/docs/user/ppl/cmd/showdatasources.rst b/docs/user/ppl/cmd/showdatasources.rst index f7c6beb82f..d954ef0c04 100644 --- a/docs/user/ppl/cmd/showdatasources.rst +++ b/docs/user/ppl/cmd/showdatasources.rst @@ -28,9 +28,9 @@ PPL query for all PROMETHEUS DATASOURCES:: os> show datasources | where CONNECTOR_TYPE='PROMETHEUS'; fetched rows / total rows = 1/1 - +-------------------+------------------+ - | DATASOURCE_NAME | CONNECTOR_TYPE | - |-------------------+------------------| - | my_prometheus | PROMETHEUS | - +-------------------+------------------+ + +-----------------+----------------+ + | DATASOURCE_NAME | CONNECTOR_TYPE | + |-----------------+----------------| + | my_prometheus | PROMETHEUS | + +-----------------+----------------+ diff --git a/docs/user/ppl/cmd/sort.rst b/docs/user/ppl/cmd/sort.rst index 6a00ebc24c..377f0a5e01 100644 --- a/docs/user/ppl/cmd/sort.rst +++ b/docs/user/ppl/cmd/sort.rst @@ -32,14 +32,14 @@ PPL query:: os> source=accounts | sort age | fields account_number, age; fetched rows / total rows = 4/4 - +------------------+-------+ - | account_number | age | - |------------------+-------| - | 13 | 28 | - | 1 | 32 | - | 18 | 33 | - | 6 | 36 | - +------------------+-------+ + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 13 | 28 | + | 1 | 32 | + | 18 | 33 | + | 6 | 36 | + +----------------+-----+ Example 2: Sort by one field return all the result @@ -51,14 +51,14 @@ PPL query:: os> source=accounts | sort age | fields account_number, age; fetched rows / total rows = 4/4 - +------------------+-------+ - | account_number | age | - |------------------+-------| - | 13 | 28 | - | 1 | 32 | - | 18 | 33 | - | 6 | 36 | - +------------------+-------+ + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 13 | 28 | + | 1 | 32 | + | 18 | 33 | + | 6 | 36 | + +----------------+-----+ Example 3: Sort by one field in descending order @@ -70,14 +70,14 @@ PPL query:: os> source=accounts | sort - age | fields account_number, age; fetched rows / total rows = 4/4 - +------------------+-------+ - | account_number | age | - |------------------+-------| - | 6 | 36 | - | 18 | 33 | - | 1 | 32 | - | 13 | 28 | - +------------------+-------+ + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 6 | 36 | + | 18 | 33 | + | 1 | 32 | + | 13 | 28 | + +----------------+-----+ Example 4: Sort by multiple field ============================= @@ -88,14 +88,14 @@ PPL query:: os> source=accounts | sort + gender, - age | fields account_number, gender, age; fetched rows / total rows = 4/4 - +------------------+----------+-------+ - | account_number | gender | age | - |------------------+----------+-------| - | 13 | F | 28 | - | 6 | M | 36 | - | 18 | M | 33 | - | 1 | M | 32 | - +------------------+----------+-------+ + +----------------+--------+-----+ + | account_number | gender | age | + |----------------+--------+-----| + | 13 | F | 28 | + | 6 | M | 36 | + | 18 | M | 33 | + | 1 | M | 32 | + +----------------+--------+-----+ Example 4: Sort by field include null value =========================================== @@ -106,11 +106,11 @@ PPL query:: os> source=accounts | sort employer | fields employer; fetched rows / total rows = 4/4 - +------------+ - | employer | - |------------| - | null | - | Netagy | - | Pyrami | - | Quility | - +------------+ + +----------+ + | employer | + |----------| + | null | + | Netagy | + | Pyrami | + | Quility | + +----------+ diff --git a/docs/user/ppl/cmd/stats.rst b/docs/user/ppl/cmd/stats.rst index 19f5069bba..7d5da804ce 100644 --- a/docs/user/ppl/cmd/stats.rst +++ b/docs/user/ppl/cmd/stats.rst @@ -86,11 +86,11 @@ Example:: os> source=accounts | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 4 | - +-----------+ + +---------+ + | count() | + |---------| + | 4 | + +---------+ SUM --- @@ -104,12 +104,12 @@ Example:: os> source=accounts | stats sum(age) by gender; fetched rows / total rows = 2/2 - +------------+----------+ - | sum(age) | gender | - |------------+----------| - | 28 | F | - | 101 | M | - +------------+----------+ + +----------+--------+ + | sum(age) | gender | + |----------+--------| + | 28 | F | + | 101 | M | + +----------+--------+ AVG --- @@ -123,12 +123,12 @@ Example:: os> source=accounts | stats avg(age) by gender; fetched rows / total rows = 2/2 - +--------------------+----------+ - | avg(age) | gender | - |--------------------+----------| - | 28.0 | F | - | 33.666666666666664 | M | - +--------------------+----------+ + +--------------------+--------+ + | avg(age) | gender | + |--------------------+--------| + | 28.0 | F | + | 33.666666666666664 | M | + +--------------------+--------+ MAX --- @@ -142,11 +142,11 @@ Example:: os> source=accounts | stats max(age); fetched rows / total rows = 1/1 - +------------+ - | max(age) | - |------------| - | 36 | - +------------+ + +----------+ + | max(age) | + |----------| + | 36 | + +----------+ MIN --- @@ -160,11 +160,11 @@ Example:: os> source=accounts | stats min(age); fetched rows / total rows = 1/1 - +------------+ - | min(age) | - |------------| - | 28 | - +------------+ + +----------+ + | min(age) | + |----------| + | 28 | + +----------+ VAR_SAMP -------- @@ -196,11 +196,11 @@ Example:: os> source=accounts | stats var_pop(age); fetched rows / total rows = 1/1 - +----------------+ - | var_pop(age) | - |----------------| - | 8.1875 | - +----------------+ + +--------------+ + | var_pop(age) | + |--------------| + | 8.1875 | + +--------------+ STDDEV_SAMP ----------- @@ -214,11 +214,11 @@ Example:: os> source=accounts | stats stddev_samp(age); fetched rows / total rows = 1/1 - +--------------------+ - | stddev_samp(age) | - |--------------------| - | 3.304037933599835 | - +--------------------+ + +-------------------+ + | stddev_samp(age) | + |-------------------| + | 3.304037933599835 | + +-------------------+ STDDEV_POP ---------- @@ -273,12 +273,12 @@ Example:: os> source=accounts | stats percentile(age, 90) by gender; fetched rows / total rows = 2/2 - +-----------------------+----------+ - | percentile(age, 90) | gender | - |-----------------------+----------| - | 28 | F | - | 36 | M | - +-----------------------+----------+ + +---------------------+--------+ + | percentile(age, 90) | gender | + |---------------------+--------| + | 28 | F | + | 36 | M | + +---------------------+--------+ Example 1: Calculate the count of events ======================================== @@ -289,11 +289,11 @@ PPL query:: os> source=accounts | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 4 | - +-----------+ + +---------+ + | count() | + |---------| + | 4 | + +---------+ Example 2: Calculate the average of a field @@ -305,11 +305,11 @@ PPL query:: os> source=accounts | stats avg(age); fetched rows / total rows = 1/1 - +------------+ - | avg(age) | - |------------| - | 32.25 | - +------------+ + +----------+ + | avg(age) | + |----------| + | 32.25 | + +----------+ Example 3: Calculate the average of a field by group @@ -321,12 +321,12 @@ PPL query:: os> source=accounts | stats avg(age) by gender; fetched rows / total rows = 2/2 - +--------------------+----------+ - | avg(age) | gender | - |--------------------+----------| - | 28.0 | F | - | 33.666666666666664 | M | - +--------------------+----------+ + +--------------------+--------+ + | avg(age) | gender | + |--------------------+--------| + | 28.0 | F | + | 33.666666666666664 | M | + +--------------------+--------+ Example 4: Calculate the average, sum and count of a field by group @@ -338,12 +338,12 @@ PPL query:: os> source=accounts | stats avg(age), sum(age), count() by gender; fetched rows / total rows = 2/2 - +--------------------+------------+-----------+----------+ - | avg(age) | sum(age) | count() | gender | - |--------------------+------------+-----------+----------| - | 28.0 | 28 | 1 | F | - | 33.666666666666664 | 101 | 3 | M | - +--------------------+------------+-----------+----------+ + +--------------------+----------+---------+--------+ + | avg(age) | sum(age) | count() | gender | + |--------------------+----------+---------+--------| + | 28.0 | 28 | 1 | F | + | 33.666666666666664 | 101 | 3 | M | + +--------------------+----------+---------+--------+ Example 5: Calculate the maximum of a field =========================================== @@ -354,11 +354,11 @@ PPL query:: os> source=accounts | stats max(age); fetched rows / total rows = 1/1 - +------------+ - | max(age) | - |------------| - | 36 | - +------------+ + +----------+ + | max(age) | + |----------| + | 36 | + +----------+ Example 6: Calculate the maximum and minimum of a field by group ================================================================ @@ -369,12 +369,12 @@ PPL query:: os> source=accounts | stats max(age), min(age) by gender; fetched rows / total rows = 2/2 - +------------+------------+----------+ - | max(age) | min(age) | gender | - |------------+------------+----------| - | 28 | 28 | F | - | 36 | 32 | M | - +------------+------------+----------+ + +----------+----------+--------+ + | max(age) | min(age) | gender | + |----------+----------+--------| + | 28 | 28 | F | + | 36 | 32 | M | + +----------+----------+--------+ Example 7: Calculate the distinct count of a field ================================================== @@ -385,11 +385,11 @@ PPL query:: os> source=accounts | stats count(gender), distinct_count(gender); fetched rows / total rows = 1/1 - +-----------------+--------------------------+ - | count(gender) | distinct_count(gender) | - |-----------------+--------------------------| - | 4 | 2 | - +-----------------+--------------------------+ + +---------------+------------------------+ + | count(gender) | distinct_count(gender) | + |---------------+------------------------| + | 4 | 2 | + +---------------+------------------------+ Example 8: Calculate the count by a span ======================================== @@ -400,12 +400,12 @@ PPL query:: os> source=accounts | stats count(age) by span(age, 10) as age_span fetched rows / total rows = 2/2 - +--------------+------------+ - | count(age) | age_span | - |--------------+------------| - | 1 | 20 | - | 3 | 30 | - +--------------+------------+ + +------------+----------+ + | count(age) | age_span | + |------------+----------| + | 1 | 20 | + | 3 | 30 | + +------------+----------+ Example 9: Calculate the count by a gender and span =================================================== @@ -416,13 +416,13 @@ PPL query:: os> source=accounts | stats count() as cnt by span(age, 5) as age_span, gender fetched rows / total rows = 3/3 - +-------+------------+----------+ - | cnt | age_span | gender | - |-------+------------+----------| - | 1 | 25 | F | - | 2 | 30 | M | - | 1 | 35 | M | - +-------+------------+----------+ + +-----+----------+--------+ + | cnt | age_span | gender | + |-----+----------+--------| + | 1 | 25 | F | + | 2 | 30 | M | + | 1 | 35 | M | + +-----+----------+--------+ Span will always be the first grouping key whatever order you specify. @@ -430,13 +430,13 @@ PPL query:: os> source=accounts | stats count() as cnt by gender, span(age, 5) as age_span fetched rows / total rows = 3/3 - +-------+------------+----------+ - | cnt | age_span | gender | - |-------+------------+----------| - | 1 | 25 | F | - | 2 | 30 | M | - | 1 | 35 | M | - +-------+------------+----------+ + +-----+----------+--------+ + | cnt | age_span | gender | + |-----+----------+--------| + | 1 | 25 | F | + | 2 | 30 | M | + | 1 | 35 | M | + +-----+----------+--------+ Example 10: Calculate the count and get email list by a gender and span ======================================================================= @@ -447,13 +447,13 @@ PPL query:: os> source=accounts | stats count() as cnt, take(email, 5) by span(age, 5) as age_span, gender fetched rows / total rows = 3/3 - +-------+--------------------------------------------+------------+----------+ - | cnt | take(email, 5) | age_span | gender | - |-------+--------------------------------------------+------------+----------| - | 1 | [] | 25 | F | - | 2 | [amberduke@pyrami.com,daleadams@boink.com] | 30 | M | - | 1 | [hattiebond@netagy.com] | 35 | M | - +-------+--------------------------------------------+------------+----------+ + +-----+--------------------------------------------+----------+--------+ + | cnt | take(email, 5) | age_span | gender | + |-----+--------------------------------------------+----------+--------| + | 1 | [] | 25 | F | + | 2 | [amberduke@pyrami.com,daleadams@boink.com] | 30 | M | + | 1 | [hattiebond@netagy.com] | 35 | M | + +-----+--------------------------------------------+----------+--------+ Example 11: Calculate the percentile of a field =============================================== @@ -464,11 +464,11 @@ PPL query:: os> source=accounts | stats percentile(age, 90); fetched rows / total rows = 1/1 - +-----------------------+ - | percentile(age, 90) | - |-----------------------| - | 36 | - +-----------------------+ + +---------------------+ + | percentile(age, 90) | + |---------------------| + | 36 | + +---------------------+ Example 12: Calculate the percentile of a field by group @@ -480,12 +480,12 @@ PPL query:: os> source=accounts | stats percentile(age, 90) by gender; fetched rows / total rows = 2/2 - +-----------------------+----------+ - | percentile(age, 90) | gender | - |-----------------------+----------| - | 28 | F | - | 36 | M | - +-----------------------+----------+ + +---------------------+--------+ + | percentile(age, 90) | gender | + |---------------------+--------| + | 28 | F | + | 36 | M | + +---------------------+--------+ Example 13: Calculate the percentile by a gender and span ========================================================= @@ -496,10 +496,10 @@ PPL query:: os> source=accounts | stats percentile(age, 90) as p90 by span(age, 10) as age_span, gender fetched rows / total rows = 2/2 - +-------+------------+----------+ - | p90 | age_span | gender | - |-------+------------+----------| - | 28 | 20 | F | - | 36 | 30 | M | - +-------+------------+----------+ + +-----+----------+--------+ + | p90 | age_span | gender | + |-----+----------+--------| + | 28 | 20 | F | + | 36 | 30 | M | + +-----+----------+--------+ diff --git a/docs/user/ppl/cmd/top.rst b/docs/user/ppl/cmd/top.rst index cbab675d09..6fa4d9cdb0 100644 --- a/docs/user/ppl/cmd/top.rst +++ b/docs/user/ppl/cmd/top.rst @@ -32,12 +32,12 @@ PPL query:: os> source=accounts | top gender; fetched rows / total rows = 2/2 - +----------+ - | gender | - |----------| - | M | - | F | - +----------+ + +--------+ + | gender | + |--------| + | M | + | F | + +--------+ Example 2: Find the most common values in a field =========================================== @@ -48,11 +48,11 @@ PPL query:: os> source=accounts | top 1 gender; fetched rows / total rows = 1/1 - +----------+ - | gender | - |----------| - | M | - +----------+ + +--------+ + | gender | + |--------| + | M | + +--------+ Example 2: Find the most common values organized by gender ==================================================== @@ -63,12 +63,12 @@ PPL query:: os> source=accounts | top 1 age by gender; fetched rows / total rows = 2/2 - +----------+-------+ - | gender | age | - |----------+-------| - | F | 28 | - | M | 32 | - +----------+-------+ + +--------+-----+ + | gender | age | + |--------+-----| + | F | 28 | + | M | 32 | + +--------+-----+ Limitation ========== diff --git a/docs/user/ppl/cmd/where.rst b/docs/user/ppl/cmd/where.rst index 4d8718d69f..115bffe7de 100644 --- a/docs/user/ppl/cmd/where.rst +++ b/docs/user/ppl/cmd/where.rst @@ -29,10 +29,10 @@ PPL query:: os> source=accounts | where account_number=1 or gender="F" | fields account_number, gender; fetched rows / total rows = 2/2 - +------------------+----------+ - | account_number | gender | - |------------------+----------| - | 1 | M | - | 13 | F | - +------------------+----------+ + +----------------+--------+ + | account_number | gender | + |----------------+--------| + | 1 | M | + | 13 | F | + +----------------+--------+ diff --git a/docs/user/ppl/functions/condition.rst b/docs/user/ppl/functions/condition.rst index e48d4cb75c..96c3e64e72 100644 --- a/docs/user/ppl/functions/condition.rst +++ b/docs/user/ppl/functions/condition.rst @@ -24,14 +24,14 @@ Example:: os> source=accounts | eval result = isnull(employer) | fields result, employer, firstname fetched rows / total rows = 4/4 - +----------+------------+-------------+ - | result | employer | firstname | - |----------+------------+-------------| - | False | Pyrami | Amber | - | False | Netagy | Hattie | - | False | Quility | Nanette | - | True | null | Dale | - +----------+------------+-------------+ + +--------+----------+-----------+ + | result | employer | firstname | + |--------+----------+-----------| + | False | Pyrami | Amber | + | False | Netagy | Hattie | + | False | Quility | Nanette | + | True | null | Dale | + +--------+----------+-----------+ ISNOTNULL --------- @@ -49,11 +49,11 @@ Example:: os> source=accounts | where not isnotnull(employer) | fields account_number, employer fetched rows / total rows = 1/1 - +------------------+------------+ - | account_number | employer | - |------------------+------------| - | 18 | null | - +------------------+------------+ + +----------------+----------+ + | account_number | employer | + |----------------+----------| + | 18 | null | + +----------------+----------+ EXISTS ------ @@ -64,11 +64,11 @@ Example, the account 13 doesn't have email field:: os> source=accounts | where isnull(email) | fields account_number, email fetched rows / total rows = 1/1 - +------------------+---------+ - | account_number | email | - |------------------+---------| - | 13 | null | - +------------------+---------+ + +----------------+-------+ + | account_number | email | + |----------------+-------| + | 13 | null | + +----------------+-------+ IFNULL ------ @@ -86,14 +86,14 @@ Example:: os> source=accounts | eval result = ifnull(employer, 'default') | fields result, employer, firstname fetched rows / total rows = 4/4 - +----------+------------+-------------+ - | result | employer | firstname | - |----------+------------+-------------| - | Pyrami | Pyrami | Amber | - | Netagy | Netagy | Hattie | - | Quility | Quility | Nanette | - | default | null | Dale | - +----------+------------+-------------+ + +---------+----------+-----------+ + | result | employer | firstname | + |---------+----------+-----------| + | Pyrami | Pyrami | Amber | + | Netagy | Netagy | Hattie | + | Quility | Quility | Nanette | + | default | null | Dale | + +---------+----------+-----------+ NULLIF ------ @@ -111,14 +111,14 @@ Example:: os> source=accounts | eval result = nullif(employer, 'Pyrami') | fields result, employer, firstname fetched rows / total rows = 4/4 - +----------+------------+-------------+ - | result | employer | firstname | - |----------+------------+-------------| - | null | Pyrami | Amber | - | Netagy | Netagy | Hattie | - | Quility | Quility | Nanette | - | null | null | Dale | - +----------+------------+-------------+ + +---------+----------+-----------+ + | result | employer | firstname | + |---------+----------+-----------| + | null | Pyrami | Amber | + | Netagy | Netagy | Hattie | + | Quility | Quility | Nanette | + | null | null | Dale | + +---------+----------+-----------+ ISNULL @@ -137,14 +137,14 @@ Example:: os> source=accounts | eval result = isnull(employer) | fields result, employer, firstname fetched rows / total rows = 4/4 - +----------+------------+-------------+ - | result | employer | firstname | - |----------+------------+-------------| - | False | Pyrami | Amber | - | False | Netagy | Hattie | - | False | Quility | Nanette | - | True | null | Dale | - +----------+------------+-------------+ + +--------+----------+-----------+ + | result | employer | firstname | + |--------+----------+-----------| + | False | Pyrami | Amber | + | False | Netagy | Hattie | + | False | Quility | Nanette | + | True | null | Dale | + +--------+----------+-----------+ IF ------ @@ -162,33 +162,33 @@ Example:: os> source=accounts | eval result = if(true, firstname, lastname) | fields result, firstname, lastname fetched rows / total rows = 4/4 - +----------+-------------+------------+ - | result | firstname | lastname | - |----------+-------------+------------| - | Amber | Amber | Duke | - | Hattie | Hattie | Bond | - | Nanette | Nanette | Bates | - | Dale | Dale | Adams | - +----------+-------------+------------+ + +---------+-----------+----------+ + | result | firstname | lastname | + |---------+-----------+----------| + | Amber | Amber | Duke | + | Hattie | Hattie | Bond | + | Nanette | Nanette | Bates | + | Dale | Dale | Adams | + +---------+-----------+----------+ os> source=accounts | eval result = if(false, firstname, lastname) | fields result, firstname, lastname fetched rows / total rows = 4/4 - +----------+-------------+------------+ - | result | firstname | lastname | - |----------+-------------+------------| - | Duke | Amber | Duke | - | Bond | Hattie | Bond | - | Bates | Nanette | Bates | - | Adams | Dale | Adams | - +----------+-------------+------------+ + +--------+-----------+----------+ + | result | firstname | lastname | + |--------+-----------+----------| + | Duke | Amber | Duke | + | Bond | Hattie | Bond | + | Bates | Nanette | Bates | + | Adams | Dale | Adams | + +--------+-----------+----------+ os> source=accounts | eval is_vip = if(age > 30 AND isnotnull(employer), true, false) | fields is_vip, firstname, lastname fetched rows / total rows = 4/4 - +----------+-------------+------------+ - | is_vip | firstname | lastname | - |----------+-------------+------------| - | True | Amber | Duke | - | True | Hattie | Bond | - | False | Nanette | Bates | - | False | Dale | Adams | - +----------+-------------+------------+ + +--------+-----------+----------+ + | is_vip | firstname | lastname | + |--------+-----------+----------| + | True | Amber | Duke | + | True | Hattie | Bond | + | False | Nanette | Bates | + | False | Dale | Adams | + +--------+-----------+----------+ diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index a4a59a6cd1..31fb3e3cdf 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -38,21 +38,21 @@ Cast to string example:: os> source=people | eval `cbool` = CAST(true as string), `cint` = CAST(1 as string), `cdate` = CAST(CAST('2012-08-07' as date) as string) | fields `cbool`, `cint`, `cdate` fetched rows / total rows = 1/1 - +---------+--------+------------+ - | cbool | cint | cdate | - |---------+--------+------------| - | true | 1 | 2012-08-07 | - +---------+--------+------------+ + +-------+------+------------+ + | cbool | cint | cdate | + |-------+------+------------| + | true | 1 | 2012-08-07 | + +-------+------+------------+ Cast to number example:: os> source=people | eval `cbool` = CAST(true as int), `cstring` = CAST('1' as int) | fields `cbool`, `cstring` fetched rows / total rows = 1/1 - +---------+-----------+ - | cbool | cstring | - |---------+-----------| - | 1 | 1 | - +---------+-----------+ + +-------+---------+ + | cbool | cstring | + |-------+---------| + | 1 | 1 | + +-------+---------+ Cast to date example:: @@ -68,8 +68,8 @@ Cast function can be chained:: os> source=people | eval `cbool` = CAST(CAST(true as string) as boolean) | fields `cbool` fetched rows / total rows = 1/1 - +---------+ - | cbool | - |---------| - | True | - +---------+ + +-------+ + | cbool | + |-------| + | True | + +-------+ diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index f7c4091753..7665b4d66f 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -35,11 +35,11 @@ Example:: os> source=people | eval `'2020-08-26' + 1h` = ADDDATE(DATE('2020-08-26'), INTERVAL 1 HOUR), `'2020-08-26' + 1` = ADDDATE(DATE('2020-08-26'), 1), `ts '2020-08-26 01:01:01' + 1` = ADDDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) | fields `'2020-08-26' + 1h`, `'2020-08-26' + 1`, `ts '2020-08-26 01:01:01' + 1` fetched rows / total rows = 1/1 - +---------------------+--------------------+--------------------------------+ - | '2020-08-26' + 1h | '2020-08-26' + 1 | ts '2020-08-26 01:01:01' + 1 | - |---------------------+--------------------+--------------------------------| - | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | - +---------------------+--------------------+--------------------------------+ + +---------------------+------------------+------------------------------+ + | '2020-08-26' + 1h | '2020-08-26' + 1 | ts '2020-08-26 01:01:01' + 1 | + |---------------------+------------------+------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | + +---------------------+------------------+------------------------------+ @@ -73,35 +73,35 @@ Example:: os> source=people | eval `'23:59:59' + 0` = ADDTIME(TIME('23:59:59'), DATE('2004-01-01')) | fields `'23:59:59' + 0` fetched rows / total rows = 1/1 - +------------------+ - | '23:59:59' + 0 | - |------------------| - | 23:59:59 | - +------------------+ + +----------------+ + | '23:59:59' + 0 | + |----------------| + | 23:59:59 | + +----------------+ os> source=people | eval `'2004-01-01' + '23:59:59'` = ADDTIME(DATE('2004-01-01'), TIME('23:59:59')) | fields `'2004-01-01' + '23:59:59'` fetched rows / total rows = 1/1 - +-----------------------------+ - | '2004-01-01' + '23:59:59' | - |-----------------------------| - | 2004-01-01 23:59:59 | - +-----------------------------+ - - os> source=people | eval `'10:20:30' + '00:05:42'` = ADDTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' + '00:05:42'` - fetched rows / total rows = 1/1 +---------------------------+ - | '10:20:30' + '00:05:42' | + | '2004-01-01' + '23:59:59' | |---------------------------| - | 10:26:12 | + | 2004-01-01 23:59:59 | +---------------------------+ + os> source=people | eval `'10:20:30' + '00:05:42'` = ADDTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' + '00:05:42'` + fetched rows / total rows = 1/1 + +-------------------------+ + | '10:20:30' + '00:05:42' | + |-------------------------| + | 10:26:12 | + +-------------------------+ + os> source=people | eval `'2007-02-28 10:20:30' + '20:40:50'` = ADDTIME(TIMESTAMP('2007-02-28 10:20:30'), DATETIME('2002-03-04 20:40:50')) | fields `'2007-02-28 10:20:30' + '20:40:50'` fetched rows / total rows = 1/1 - +--------------------------------------+ - | '2007-02-28 10:20:30' + '20:40:50' | - |--------------------------------------| - | 2007-03-01 07:01:20 | - +--------------------------------------+ + +------------------------------------+ + | '2007-02-28 10:20:30' + '20:40:50' | + |------------------------------------| + | 2007-03-01 07:01:20 | + +------------------------------------+ CONVERT_TZ @@ -121,121 +121,121 @@ Example:: os> source=people | eval `convert_tz('2008-05-15 12:00:00','+00:00','+10:00')` = convert_tz('2008-05-15 12:00:00','+00:00','+10:00') | fields `convert_tz('2008-05-15 12:00:00','+00:00','+10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-05-15 12:00:00','+00:00','+10:00') | - |-------------------------------------------------------| - | 2008-05-15 22:00:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-05-15 12:00:00','+00:00','+10:00') | + |-----------------------------------------------------| + | 2008-05-15 22:00:00 | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range, such as +15:00 in this example will return null. Example:: os> source=people | eval `convert_tz('2008-05-15 12:00:00','+00:00','+15:00')` = convert_tz('2008-05-15 12:00:00','+00:00','+15:00')| fields `convert_tz('2008-05-15 12:00:00','+00:00','+15:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-05-15 12:00:00','+00:00','+15:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-05-15 12:00:00','+00:00','+15:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ Conversion from a positive timezone to a negative timezone that goes over date line. Example:: os> source=people | eval `convert_tz('2008-05-15 12:00:00','+03:30','-10:00')` = convert_tz('2008-05-15 12:00:00','+03:30','-10:00') | fields `convert_tz('2008-05-15 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-05-15 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | 2008-05-14 22:30:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-05-15 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | 2008-05-14 22:30:00 | + +-----------------------------------------------------+ Valid dates are required in convert_tz, invalid dates such as April 31st (not a date in the Gregorian calendar) will result in null. Example:: os> source=people | eval `convert_tz('2008-04-31 12:00:00','+03:30','-10:00')` = convert_tz('2008-04-31 12:00:00','+03:30','-10:00') | fields `convert_tz('2008-04-31 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-04-31 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-04-31 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ Valid dates are required in convert_tz, invalid dates such as February 30th (not a date in the Gregorian calendar) will result in null. Example:: os> source=people | eval `convert_tz('2008-02-30 12:00:00','+03:30','-10:00')` = convert_tz('2008-02-30 12:00:00','+03:30','-10:00') | fields `convert_tz('2008-02-30 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-30 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-30 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ February 29th 2008 is a valid date because it is a leap year. Example:: os> source=people | eval `convert_tz('2008-02-29 12:00:00','+03:30','-10:00')` = convert_tz('2008-02-29 12:00:00','+03:30','-10:00') | fields `convert_tz('2008-02-29 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-29 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | 2008-02-28 22:30:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-29 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | 2008-02-28 22:30:00 | + +-----------------------------------------------------+ Valid dates are required in convert_tz, invalid dates such as February 29th 2007 (2007 is not a leap year) will result in null. Example:: os> source=people | eval `convert_tz('2007-02-29 12:00:00','+03:30','-10:00')` = convert_tz('2007-02-29 12:00:00','+03:30','-10:00') | fields `convert_tz('2007-02-29 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2007-02-29 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2007-02-29 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range, such as +14:01 in this example will return null. Example:: os> source=people | eval `convert_tz('2008-02-01 12:00:00','+14:01','+00:00')` = convert_tz('2008-02-01 12:00:00','+14:01','+00:00') | fields `convert_tz('2008-02-01 12:00:00','+14:01','+00:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-01 12:00:00','+14:01','+00:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-01 12:00:00','+14:01','+00:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range, such as +14:00 in this example will return a correctly converted date time object. Example:: os> source=people | eval `convert_tz('2008-02-01 12:00:00','+14:00','+00:00')` = convert_tz('2008-02-01 12:00:00','+14:00','+00:00') | fields `convert_tz('2008-02-01 12:00:00','+14:00','+00:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-01 12:00:00','+14:00','+00:00') | - |-------------------------------------------------------| - | 2008-01-31 22:00:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-01 12:00:00','+14:00','+00:00') | + |-----------------------------------------------------| + | 2008-01-31 22:00:00 | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range, such as -14:00 will result in null Example:: os> source=people | eval `convert_tz('2008-02-01 12:00:00','-14:00','+00:00')` = convert_tz('2008-02-01 12:00:00','-14:00','+00:00') | fields `convert_tz('2008-02-01 12:00:00','-14:00','+00:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-01 12:00:00','-14:00','+00:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-01 12:00:00','-14:00','+00:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. This timezone is within range so it is valid and will convert the time. Example:: os> source=people | eval `convert_tz('2008-02-01 12:00:00','-13:59','+00:00')` = convert_tz('2008-02-01 12:00:00','-13:59','+00:00') | fields `convert_tz('2008-02-01 12:00:00','-13:59','+00:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-01 12:00:00','-13:59','+00:00') | - |-------------------------------------------------------| - | 2008-02-02 01:59:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-01 12:00:00','-13:59','+00:00') | + |-----------------------------------------------------| + | 2008-02-02 01:59:00 | + +-----------------------------------------------------+ CURDATE @@ -255,11 +255,11 @@ Example:: > source=people | eval `CURDATE()` = CURDATE() | fields `CURDATE()` fetched rows / total rows = 1/1 - +-------------+ - | CURDATE() | - |-------------| - | 2022-08-02 | - +-------------+ + +------------+ + | CURDATE() | + |------------| + | 2022-08-02 | + +------------+ CURRENT_DATE @@ -336,11 +336,11 @@ Example:: > source=people | eval `value_1` = CURTIME(), `value_2` = CURTIME() | fields `value_1`, `value_2` fetched rows / total rows = 1/1 - +-----------+-----------+ - | value_1 | value_2 | - |-----------+-----------| - | 15:39:05 | 15:39:05 | - +-----------+-----------+ + +----------+----------+ + | value_1 | value_2 | + |----------+----------| + | 15:39:05 | 15:39:05 | + +----------+----------+ DATE @@ -359,35 +359,35 @@ Example:: os> source=people | eval `DATE('2020-08-26')` = DATE('2020-08-26') | fields `DATE('2020-08-26')` fetched rows / total rows = 1/1 - +----------------------+ - | DATE('2020-08-26') | - |----------------------| - | 2020-08-26 | - +----------------------+ + +--------------------+ + | DATE('2020-08-26') | + |--------------------| + | 2020-08-26 | + +--------------------+ os> source=people | eval `DATE(TIMESTAMP('2020-08-26 13:49:00'))` = DATE(TIMESTAMP('2020-08-26 13:49:00')) | fields `DATE(TIMESTAMP('2020-08-26 13:49:00'))` fetched rows / total rows = 1/1 - +------------------------------------------+ - | DATE(TIMESTAMP('2020-08-26 13:49:00')) | - |------------------------------------------| - | 2020-08-26 | - +------------------------------------------+ + +----------------------------------------+ + | DATE(TIMESTAMP('2020-08-26 13:49:00')) | + |----------------------------------------| + | 2020-08-26 | + +----------------------------------------+ os> source=people | eval `DATE('2020-08-26 13:49')` = DATE('2020-08-26 13:49') | fields `DATE('2020-08-26 13:49')` fetched rows / total rows = 1/1 - +----------------------------+ - | DATE('2020-08-26 13:49') | - |----------------------------| - | 2020-08-26 | - +----------------------------+ + +--------------------------+ + | DATE('2020-08-26 13:49') | + |--------------------------| + | 2020-08-26 | + +--------------------------+ os> source=people | eval `DATE('2020-08-26 13:49')` = DATE('2020-08-26 13:49') | fields `DATE('2020-08-26 13:49')` fetched rows / total rows = 1/1 - +----------------------------+ - | DATE('2020-08-26 13:49') | - |----------------------------| - | 2020-08-26 | - +----------------------------+ + +--------------------------+ + | DATE('2020-08-26 13:49') | + |--------------------------| + | 2020-08-26 | + +--------------------------+ DATE_ADD @@ -410,11 +410,11 @@ Example:: os> source=people | eval `'2020-08-26' + 1h` = DATE_ADD(DATE('2020-08-26'), INTERVAL 1 HOUR), `ts '2020-08-26 01:01:01' + 1d` = DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 1 DAY) | fields `'2020-08-26' + 1h`, `ts '2020-08-26 01:01:01' + 1d` fetched rows / total rows = 1/1 - +---------------------+---------------------------------+ - | '2020-08-26' + 1h | ts '2020-08-26 01:01:01' + 1d | - |---------------------+---------------------------------| - | 2020-08-26 01:00:00 | 2020-08-27 01:01:01 | - +---------------------+---------------------------------+ + +---------------------+-------------------------------+ + | '2020-08-26' + 1h | ts '2020-08-26 01:01:01' + 1d | + |---------------------+-------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 01:01:01 | + +---------------------+-------------------------------+ DATE_FORMAT @@ -509,11 +509,11 @@ Example:: os> source=people | eval `DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f')` = DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f'), `DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r')` = DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | fields `DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f')`, `DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r')` fetched rows / total rows = 1/1 - +------------------------------------------------------+-----------------------------------------------------------------------+ - | DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f') | DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | - |------------------------------------------------------+-----------------------------------------------------------------------| - | 13:14:15.012345 | 1998-Jan-31st 01:14:15 PM | - +------------------------------------------------------+-----------------------------------------------------------------------+ + +----------------------------------------------------+---------------------------------------------------------------------+ + | DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f') | DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | + |----------------------------------------------------+---------------------------------------------------------------------| + | 13:14:15.012345 | 1998-Jan-31st 01:14:15 PM | + +----------------------------------------------------+---------------------------------------------------------------------+ DATETIME @@ -538,11 +538,11 @@ Example:: os> source=people | eval `DATETIME('2004-02-28 23:00:00-10:00', '+10:00')` = DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | fields `DATETIME('2004-02-28 23:00:00-10:00', '+10:00')` fetched rows / total rows = 1/1 - +---------------------------------------------------+ - | DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | - |---------------------------------------------------| - | 2004-02-29 19:00:00 | - +---------------------------------------------------+ + +-------------------------------------------------+ + | DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | + |-------------------------------------------------| + | 2004-02-29 19:00:00 | + +-------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. @@ -550,22 +550,22 @@ Example:: os> source=people | eval `DATETIME('2008-01-01 02:00:00', '-14:00')` = DATETIME('2008-01-01 02:00:00', '-14:00') | fields `DATETIME('2008-01-01 02:00:00', '-14:00')` fetched rows / total rows = 1/1 - +---------------------------------------------+ - | DATETIME('2008-01-01 02:00:00', '-14:00') | - |---------------------------------------------| - | null | - +---------------------------------------------+ + +-------------------------------------------+ + | DATETIME('2008-01-01 02:00:00', '-14:00') | + |-------------------------------------------| + | null | + +-------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. Example:: os> source=people | eval `DATETIME('2008-02-30 02:00:00', '-00:00')` = DATETIME('2008-02-30 02:00:00', '-00:00') | fields `DATETIME('2008-02-30 02:00:00', '-00:00')` fetched rows / total rows = 1/1 - +---------------------------------------------+ - | DATETIME('2008-02-30 02:00:00', '-00:00') | - |---------------------------------------------| - | null | - +---------------------------------------------+ + +-------------------------------------------+ + | DATETIME('2008-02-30 02:00:00', '-00:00') | + |-------------------------------------------| + | null | + +-------------------------------------------+ DATE_SUB @@ -588,11 +588,11 @@ Example:: os> source=people | eval `'2008-01-02' - 31d` = DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY), `ts '2020-08-26 01:01:01' + 1h` = DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 1 HOUR) | fields `'2008-01-02' - 31d`, `ts '2020-08-26 01:01:01' + 1h` fetched rows / total rows = 1/1 - +----------------------+---------------------------------+ - | '2008-01-02' - 31d | ts '2020-08-26 01:01:01' + 1h | - |----------------------+---------------------------------| - | 2007-12-02 00:00:00 | 2020-08-26 00:01:01 | - +----------------------+---------------------------------+ + +---------------------+-------------------------------+ + | '2008-01-02' - 31d | ts '2020-08-26 01:01:01' + 1h | + |---------------------+-------------------------------| + | 2007-12-02 00:00:00 | 2020-08-26 00:01:01 | + +---------------------+-------------------------------+ DATEDIFF @@ -608,11 +608,11 @@ Example:: os> source=people | eval `'2000-01-02' - '2000-01-01'` = DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59')), `'2001-02-01' - '2004-01-01'` = DATEDIFF(DATE('2001-02-01'), TIMESTAMP('2004-01-01 00:00:00')), `today - today` = DATEDIFF(TIME('23:59:59'), TIME('00:00:00')) | fields `'2000-01-02' - '2000-01-01'`, `'2001-02-01' - '2004-01-01'`, `today - today` fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+-----------------+ - | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | - |-------------------------------+-------------------------------+-----------------| - | 1 | -1064 | 0 | - +-------------------------------+-------------------------------+-----------------+ + +-----------------------------+-----------------------------+---------------+ + | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | + |-----------------------------+-----------------------------+---------------| + | 1 | -1064 | 0 | + +-----------------------------+-----------------------------+---------------+ DAY @@ -633,11 +633,11 @@ Example:: os> source=people | eval `DAY(DATE('2020-08-26'))` = DAY(DATE('2020-08-26')) | fields `DAY(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------+ - | DAY(DATE('2020-08-26')) | - |---------------------------| - | 26 | - +---------------------------+ + +-------------------------+ + | DAY(DATE('2020-08-26')) | + |-------------------------| + | 26 | + +-------------------------+ DAYNAME @@ -656,11 +656,11 @@ Example:: os> source=people | eval `DAYNAME(DATE('2020-08-26'))` = DAYNAME(DATE('2020-08-26')) | fields `DAYNAME(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-------------------------------+ - | DAYNAME(DATE('2020-08-26')) | - |-------------------------------| - | Wednesday | - +-------------------------------+ + +-----------------------------+ + | DAYNAME(DATE('2020-08-26')) | + |-----------------------------| + | Wednesday | + +-----------------------------+ DAYOFMONTH @@ -681,11 +681,11 @@ Example:: os> source=people | eval `DAYOFMONTH(DATE('2020-08-26'))` = DAYOFMONTH(DATE('2020-08-26')) | fields `DAYOFMONTH(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +----------------------------------+ - | DAYOFMONTH(DATE('2020-08-26')) | - |----------------------------------| - | 26 | - +----------------------------------+ + +--------------------------------+ + | DAYOFMONTH(DATE('2020-08-26')) | + |--------------------------------| + | 26 | + +--------------------------------+ DAY_OF_MONTH @@ -706,11 +706,11 @@ Example:: os> source=people | eval `DAY_OF_MONTH(DATE('2020-08-26'))` = DAY_OF_MONTH(DATE('2020-08-26')) | fields `DAY_OF_MONTH(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +------------------------------------+ - | DAY_OF_MONTH(DATE('2020-08-26')) | - |------------------------------------| - | 26 | - +------------------------------------+ + +----------------------------------+ + | DAY_OF_MONTH(DATE('2020-08-26')) | + |----------------------------------| + | 26 | + +----------------------------------+ DAYOFWEEK @@ -731,11 +731,11 @@ Example:: os> source=people | eval `DAYOFWEEK(DATE('2020-08-26'))` = DAYOFWEEK(DATE('2020-08-26')) | fields `DAYOFWEEK(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | DAYOFWEEK(DATE('2020-08-26')) | - |---------------------------------| - | 4 | - +---------------------------------+ + +-------------------------------+ + | DAYOFWEEK(DATE('2020-08-26')) | + |-------------------------------| + | 4 | + +-------------------------------+ DAY_OF_WEEK @@ -756,11 +756,11 @@ Example:: os> source=people | eval `DAY_OF_WEEK(DATE('2020-08-26'))` = DAY_OF_WEEK(DATE('2020-08-26')) | fields `DAY_OF_WEEK(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-----------------------------------+ - | DAY_OF_WEEK(DATE('2020-08-26')) | - |-----------------------------------| - | 4 | - +-----------------------------------+ + +---------------------------------+ + | DAY_OF_WEEK(DATE('2020-08-26')) | + |---------------------------------| + | 4 | + +---------------------------------+ DAYOFYEAR @@ -781,11 +781,11 @@ Example:: os> source=people | eval `DAYOFYEAR(DATE('2020-08-26'))` = DAYOFYEAR(DATE('2020-08-26')) | fields `DAYOFYEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | DAYOFYEAR(DATE('2020-08-26')) | - |---------------------------------| - | 239 | - +---------------------------------+ + +-------------------------------+ + | DAYOFYEAR(DATE('2020-08-26')) | + |-------------------------------| + | 239 | + +-------------------------------+ DAY_OF_YEAR @@ -806,11 +806,11 @@ Example:: os> source=people | eval `DAY_OF_YEAR(DATE('2020-08-26'))` = DAY_OF_YEAR(DATE('2020-08-26')) | fields `DAY_OF_YEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-----------------------------------+ - | DAY_OF_YEAR(DATE('2020-08-26')) | - |-----------------------------------| - | 239 | - +-----------------------------------+ + +---------------------------------+ + | DAY_OF_YEAR(DATE('2020-08-26')) | + |---------------------------------| + | 239 | + +---------------------------------+ EXTRACT @@ -877,11 +877,11 @@ Example:: os> source=people | eval `extract(YEAR_MONTH FROM "2023-02-07 10:11:12")` = extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | fields `extract(YEAR_MONTH FROM "2023-02-07 10:11:12")` fetched rows / total rows = 1/1 - +--------------------------------------------------+ - | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | - |--------------------------------------------------| - | 202302 | - +--------------------------------------------------+ + +------------------------------------------------+ + | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | + |------------------------------------------------| + | 202302 | + +------------------------------------------------+ FROM_DAYS @@ -900,11 +900,11 @@ Example:: os> source=people | eval `FROM_DAYS(733687)` = FROM_DAYS(733687) | fields `FROM_DAYS(733687)` fetched rows / total rows = 1/1 - +---------------------+ - | FROM_DAYS(733687) | - |---------------------| - | 2008-10-07 | - +---------------------+ + +-------------------+ + | FROM_DAYS(733687) | + |-------------------| + | 2008-10-07 | + +-------------------+ FROM_UNIXTIME @@ -928,19 +928,19 @@ Examples:: os> source=people | eval `FROM_UNIXTIME(1220249547)` = FROM_UNIXTIME(1220249547) | fields `FROM_UNIXTIME(1220249547)` fetched rows / total rows = 1/1 - +-----------------------------+ - | FROM_UNIXTIME(1220249547) | - |-----------------------------| - | 2008-09-01 06:12:27 | - +-----------------------------+ + +---------------------------+ + | FROM_UNIXTIME(1220249547) | + |---------------------------| + | 2008-09-01 06:12:27 | + +---------------------------+ os> source=people | eval `FROM_UNIXTIME(1220249547, '%T')` = FROM_UNIXTIME(1220249547, '%T') | fields `FROM_UNIXTIME(1220249547, '%T')` fetched rows / total rows = 1/1 - +-----------------------------------+ - | FROM_UNIXTIME(1220249547, '%T') | - |-----------------------------------| - | 06:12:27 | - +-----------------------------------+ + +---------------------------------+ + | FROM_UNIXTIME(1220249547, '%T') | + |---------------------------------| + | 06:12:27 | + +---------------------------------+ GET_FORMAT @@ -958,11 +958,11 @@ Examples:: os> source=people | eval `GET_FORMAT(DATE, 'USA')` = GET_FORMAT(DATE, 'USA') | fields `GET_FORMAT(DATE, 'USA')` fetched rows / total rows = 1/1 - +---------------------------+ - | GET_FORMAT(DATE, 'USA') | - |---------------------------| - | %m.%d.%Y | - +---------------------------+ + +-------------------------+ + | GET_FORMAT(DATE, 'USA') | + |-------------------------| + | %m.%d.%Y | + +-------------------------+ HOUR @@ -983,11 +983,11 @@ Example:: os> source=people | eval `HOUR(TIME('01:02:03'))` = HOUR(TIME('01:02:03')) | fields `HOUR(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +--------------------------+ - | HOUR(TIME('01:02:03')) | - |--------------------------| - | 1 | - +--------------------------+ + +------------------------+ + | HOUR(TIME('01:02:03')) | + |------------------------| + | 1 | + +------------------------+ HOUR_OF_DAY @@ -1008,11 +1008,11 @@ Example:: os> source=people | eval `HOUR_OF_DAY(TIME('01:02:03'))` = HOUR_OF_DAY(TIME('01:02:03')) | fields `HOUR_OF_DAY(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | HOUR_OF_DAY(TIME('01:02:03')) | - |---------------------------------| - | 1 | - +---------------------------------+ + +-------------------------------+ + | HOUR_OF_DAY(TIME('01:02:03')) | + |-------------------------------| + | 1 | + +-------------------------------+ LAST_DAY @@ -1028,11 +1028,11 @@ Example:: os> source=people | eval `last_day('2023-02-06')` = last_day('2023-02-06') | fields `last_day('2023-02-06')` fetched rows / total rows = 1/1 - +--------------------------+ - | last_day('2023-02-06') | - |--------------------------| - | 2023-02-28 | - +--------------------------+ + +------------------------+ + | last_day('2023-02-06') | + |------------------------| + | 2023-02-28 | + +------------------------+ LOCALTIMESTAMP @@ -1100,11 +1100,11 @@ Example:: os> source=people | eval `MAKEDATE(1945, 5.9)` = MAKEDATE(1945, 5.9), `MAKEDATE(1984, 1984)` = MAKEDATE(1984, 1984) | fields `MAKEDATE(1945, 5.9)`, `MAKEDATE(1984, 1984)` fetched rows / total rows = 1/1 - +-----------------------+------------------------+ - | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | - |-----------------------+------------------------| - | 1945-01-06 | 1989-06-06 | - +-----------------------+------------------------+ + +---------------------+----------------------+ + | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | + |---------------------+----------------------| + | 1945-01-06 | 1989-06-06 | + +---------------------+----------------------+ MAKETIME @@ -1132,11 +1132,11 @@ Example:: os> source=people | eval `MAKETIME(20, 30, 40)` = MAKETIME(20, 30, 40), `MAKETIME(20.2, 49.5, 42.100502)` = MAKETIME(20.2, 49.5, 42.100502) | fields `MAKETIME(20, 30, 40)`, `MAKETIME(20.2, 49.5, 42.100502)` fetched rows / total rows = 1/1 - +------------------------+-----------------------------------+ - | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | - |------------------------+-----------------------------------| - | 20:30:40 | 20:50:42.100502 | - +------------------------+-----------------------------------+ + +----------------------+---------------------------------+ + | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | + |----------------------+---------------------------------| + | 20:30:40 | 20:50:42.100502 | + +----------------------+---------------------------------+ MICROSECOND @@ -1155,11 +1155,11 @@ Example:: os> source=people | eval `MICROSECOND(TIME('01:02:03.123456'))` = MICROSECOND(TIME('01:02:03.123456')) | fields `MICROSECOND(TIME('01:02:03.123456'))` fetched rows / total rows = 1/1 - +----------------------------------------+ - | MICROSECOND(TIME('01:02:03.123456')) | - |----------------------------------------| - | 123456 | - +----------------------------------------+ + +--------------------------------------+ + | MICROSECOND(TIME('01:02:03.123456')) | + |--------------------------------------| + | 123456 | + +--------------------------------------+ MINUTE @@ -1180,11 +1180,11 @@ Example:: os> source=people | eval `MINUTE(TIME('01:02:03'))` = MINUTE(TIME('01:02:03')) | fields `MINUTE(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +----------------------------+ - | MINUTE(TIME('01:02:03')) | - |----------------------------| - | 2 | - +----------------------------+ + +--------------------------+ + | MINUTE(TIME('01:02:03')) | + |--------------------------| + | 2 | + +--------------------------+ MINUTE_OF_DAY @@ -1203,11 +1203,11 @@ Example:: os> source=people | eval `MINUTE_OF_DAY(TIME('01:02:03'))` = MINUTE_OF_DAY(TIME('01:02:03')) | fields `MINUTE_OF_DAY(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +-----------------------------------+ - | MINUTE_OF_DAY(TIME('01:02:03')) | - |-----------------------------------| - | 62 | - +-----------------------------------+ + +---------------------------------+ + | MINUTE_OF_DAY(TIME('01:02:03')) | + |---------------------------------| + | 62 | + +---------------------------------+ MINUTE_OF_HOUR @@ -1228,11 +1228,11 @@ Example:: os> source=people | eval `MINUTE_OF_HOUR(TIME('01:02:03'))` = MINUTE_OF_HOUR(TIME('01:02:03')) | fields `MINUTE_OF_HOUR(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +------------------------------------+ - | MINUTE_OF_HOUR(TIME('01:02:03')) | - |------------------------------------| - | 2 | - +------------------------------------+ + +----------------------------------+ + | MINUTE_OF_HOUR(TIME('01:02:03')) | + |----------------------------------| + | 2 | + +----------------------------------+ MONTH @@ -1253,11 +1253,11 @@ Example:: os> source=people | eval `MONTH(DATE('2020-08-26'))` = MONTH(DATE('2020-08-26')) | fields `MONTH(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-----------------------------+ - | MONTH(DATE('2020-08-26')) | - |-----------------------------| - | 8 | - +-----------------------------+ + +---------------------------+ + | MONTH(DATE('2020-08-26')) | + |---------------------------| + | 8 | + +---------------------------+ MONTH_OF_YEAR @@ -1278,11 +1278,11 @@ Example:: os> source=people | eval `MONTH_OF_YEAR(DATE('2020-08-26'))` = MONTH_OF_YEAR(DATE('2020-08-26')) | fields `MONTH_OF_YEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-------------------------------------+ - | MONTH_OF_YEAR(DATE('2020-08-26')) | - |-------------------------------------| - | 8 | - +-------------------------------------+ + +-----------------------------------+ + | MONTH_OF_YEAR(DATE('2020-08-26')) | + |-----------------------------------| + | 8 | + +-----------------------------------+ MONTHNAME @@ -1301,11 +1301,11 @@ Example:: os> source=people | eval `MONTHNAME(DATE('2020-08-26'))` = MONTHNAME(DATE('2020-08-26')) | fields `MONTHNAME(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | MONTHNAME(DATE('2020-08-26')) | - |---------------------------------| - | August | - +---------------------------------+ + +-------------------------------+ + | MONTHNAME(DATE('2020-08-26')) | + |-------------------------------| + | August | + +-------------------------------+ NOW @@ -1348,11 +1348,11 @@ Example:: os> source=people | eval `PERIOD_ADD(200801, 2)` = PERIOD_ADD(200801, 2), `PERIOD_ADD(200801, -12)` = PERIOD_ADD(200801, -12) | fields `PERIOD_ADD(200801, 2)`, `PERIOD_ADD(200801, -12)` fetched rows / total rows = 1/1 - +-------------------------+---------------------------+ - | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | - |-------------------------+---------------------------| - | 200803 | 200701 | - +-------------------------+---------------------------+ + +-----------------------+-------------------------+ + | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | + |-----------------------+-------------------------| + | 200803 | 200701 | + +-----------------------+-------------------------+ PERIOD_DIFF @@ -1371,11 +1371,11 @@ Example:: os> source=people | eval `PERIOD_DIFF(200802, 200703)` = PERIOD_DIFF(200802, 200703), `PERIOD_DIFF(200802, 201003)` = PERIOD_DIFF(200802, 201003) | fields `PERIOD_DIFF(200802, 200703)`, `PERIOD_DIFF(200802, 201003)` fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+ - | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | - |-------------------------------+-------------------------------| - | 11 | -25 | - +-------------------------------+-------------------------------+ + +-----------------------------+-----------------------------+ + | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | + |-----------------------------+-----------------------------| + | 11 | -25 | + +-----------------------------+-----------------------------+ QUARTER @@ -1394,11 +1394,11 @@ Example:: os> source=people | eval `QUARTER(DATE('2020-08-26'))` = QUARTER(DATE('2020-08-26')) | fields `QUARTER(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-------------------------------+ - | QUARTER(DATE('2020-08-26')) | - |-------------------------------| - | 3 | - +-------------------------------+ + +-----------------------------+ + | QUARTER(DATE('2020-08-26')) | + |-----------------------------| + | 3 | + +-----------------------------+ SEC_TO_TIME @@ -1420,11 +1420,11 @@ Example:: os> source=people | eval `SEC_TO_TIME(3601)` = SEC_TO_TIME(3601) | eval `SEC_TO_TIME(1234.123)` = SEC_TO_TIME(1234.123) | fields `SEC_TO_TIME(3601)`, `SEC_TO_TIME(1234.123)` fetched rows / total rows = 1/1 - +---------------------+-------------------------+ - | SEC_TO_TIME(3601) | SEC_TO_TIME(1234.123) | - |---------------------+-------------------------| - | 01:00:01 | 00:20:34.123 | - +---------------------+-------------------------+ + +-------------------+-----------------------+ + | SEC_TO_TIME(3601) | SEC_TO_TIME(1234.123) | + |-------------------+-----------------------| + | 01:00:01 | 00:20:34.123 | + +-------------------+-----------------------+ SECOND @@ -1445,11 +1445,11 @@ Example:: os> source=people | eval `SECOND(TIME('01:02:03'))` = SECOND(TIME('01:02:03')) | fields `SECOND(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +----------------------------+ - | SECOND(TIME('01:02:03')) | - |----------------------------| - | 3 | - +----------------------------+ + +--------------------------+ + | SECOND(TIME('01:02:03')) | + |--------------------------| + | 3 | + +--------------------------+ SECOND_OF_MINUTE @@ -1470,11 +1470,11 @@ Example:: os> source=people | eval `SECOND_OF_MINUTE(TIME('01:02:03'))` = SECOND_OF_MINUTE(TIME('01:02:03')) | fields `SECOND_OF_MINUTE(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +--------------------------------------+ - | SECOND_OF_MINUTE(TIME('01:02:03')) | - |--------------------------------------| - | 3 | - +--------------------------------------+ + +------------------------------------+ + | SECOND_OF_MINUTE(TIME('01:02:03')) | + |------------------------------------| + | 3 | + +------------------------------------+ STR_TO_DATE @@ -1496,11 +1496,11 @@ Example:: OS> source=people | eval `str_to_date("01,5,2013", "%d,%m,%Y")` = str_to_date("01,5,2013", "%d,%m,%Y") | fields = `str_to_date("01,5,2013", "%d,%m,%Y")` fetched rows / total rows = 1/1 - +----------------------------------------+ - | str_to_date("01,5,2013", "%d,%m,%Y") | - |----------------------------------------| - | 2013-05-01 00:00:00 | - +----------------------------------------+ + +--------------------------------------+ + | str_to_date("01,5,2013", "%d,%m,%Y") | + |--------------------------------------| + | 2013-05-01 00:00:00 | + +--------------------------------------+ SUBDATE @@ -1530,11 +1530,11 @@ Example:: os> source=people | eval `'2008-01-02' - 31d` = SUBDATE(DATE('2008-01-02'), INTERVAL 31 DAY), `'2020-08-26' - 1` = SUBDATE(DATE('2020-08-26'), 1), `ts '2020-08-26 01:01:01' - 1` = SUBDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) | fields `'2008-01-02' - 31d`, `'2020-08-26' - 1`, `ts '2020-08-26 01:01:01' - 1` fetched rows / total rows = 1/1 - +----------------------+--------------------+--------------------------------+ - | '2008-01-02' - 31d | '2020-08-26' - 1 | ts '2020-08-26 01:01:01' - 1 | - |----------------------+--------------------+--------------------------------| - | 2007-12-02 00:00:00 | 2020-08-25 | 2020-08-25 01:01:01 | - +----------------------+--------------------+--------------------------------+ + +---------------------+------------------+------------------------------+ + | '2008-01-02' - 31d | '2020-08-26' - 1 | ts '2020-08-26 01:01:01' - 1 | + |---------------------+------------------+------------------------------| + | 2007-12-02 00:00:00 | 2020-08-25 | 2020-08-25 01:01:01 | + +---------------------+------------------+------------------------------+ SUBTIME @@ -1567,35 +1567,35 @@ Example:: os> source=people | eval `'23:59:59' - 0` = SUBTIME(TIME('23:59:59'), DATE('2004-01-01')) | fields `'23:59:59' - 0` fetched rows / total rows = 1/1 - +------------------+ - | '23:59:59' - 0 | - |------------------| - | 23:59:59 | - +------------------+ + +----------------+ + | '23:59:59' - 0 | + |----------------| + | 23:59:59 | + +----------------+ os> source=people | eval `'2004-01-01' - '23:59:59'` = SUBTIME(DATE('2004-01-01'), TIME('23:59:59')) | fields `'2004-01-01' - '23:59:59'` fetched rows / total rows = 1/1 - +-----------------------------+ - | '2004-01-01' - '23:59:59' | - |-----------------------------| - | 2003-12-31 00:00:01 | - +-----------------------------+ - - os> source=people | eval `'10:20:30' - '00:05:42'` = SUBTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' - '00:05:42'` - fetched rows / total rows = 1/1 +---------------------------+ - | '10:20:30' - '00:05:42' | + | '2004-01-01' - '23:59:59' | |---------------------------| - | 10:14:48 | + | 2003-12-31 00:00:01 | +---------------------------+ + os> source=people | eval `'10:20:30' - '00:05:42'` = SUBTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' - '00:05:42'` + fetched rows / total rows = 1/1 + +-------------------------+ + | '10:20:30' - '00:05:42' | + |-------------------------| + | 10:14:48 | + +-------------------------+ + os> source=people | eval `'2007-03-01 10:20:30' - '20:40:50'` = SUBTIME(TIMESTAMP('2007-03-01 10:20:30'), DATETIME('2002-03-04 20:40:50')) | fields `'2007-03-01 10:20:30' - '20:40:50'` fetched rows / total rows = 1/1 - +--------------------------------------+ - | '2007-03-01 10:20:30' - '20:40:50' | - |--------------------------------------| - | 2007-02-28 13:39:40 | - +--------------------------------------+ + +------------------------------------+ + | '2007-03-01 10:20:30' - '20:40:50' | + |------------------------------------| + | 2007-02-28 13:39:40 | + +------------------------------------+ SYSDATE @@ -1641,35 +1641,35 @@ Example:: os> source=people | eval `TIME('13:49:00')` = TIME('13:49:00') | fields `TIME('13:49:00')` fetched rows / total rows = 1/1 - +--------------------+ - | TIME('13:49:00') | - |--------------------| - | 13:49:00 | - +--------------------+ + +------------------+ + | TIME('13:49:00') | + |------------------| + | 13:49:00 | + +------------------+ os> source=people | eval `TIME('13:49')` = TIME('13:49') | fields `TIME('13:49')` fetched rows / total rows = 1/1 - +-----------------+ - | TIME('13:49') | - |-----------------| - | 13:49:00 | - +-----------------+ + +---------------+ + | TIME('13:49') | + |---------------| + | 13:49:00 | + +---------------+ os> source=people | eval `TIME('2020-08-26 13:49:00')` = TIME('2020-08-26 13:49:00') | fields `TIME('2020-08-26 13:49:00')` fetched rows / total rows = 1/1 - +-------------------------------+ - | TIME('2020-08-26 13:49:00') | - |-------------------------------| - | 13:49:00 | - +-------------------------------+ + +-----------------------------+ + | TIME('2020-08-26 13:49:00') | + |-----------------------------| + | 13:49:00 | + +-----------------------------+ os> source=people | eval `TIME('2020-08-26 13:49')` = TIME('2020-08-26 13:49') | fields `TIME('2020-08-26 13:49')` fetched rows / total rows = 1/1 - +----------------------------+ - | TIME('2020-08-26 13:49') | - |----------------------------| - | 13:49:00 | - +----------------------------+ + +--------------------------+ + | TIME('2020-08-26 13:49') | + |--------------------------| + | 13:49:00 | + +--------------------------+ TIME_FORMAT @@ -1720,11 +1720,11 @@ Example:: os> source=people | eval `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` = TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | fields `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` fetched rows / total rows = 1/1 - +------------------------------------------------------------------------------+ - | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | - |------------------------------------------------------------------------------| - | 012345 13 01 01 14 PM 01:14:15 PM 15 15 13:14:15 | - +------------------------------------------------------------------------------+ + +----------------------------------------------------------------------------+ + | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | + |----------------------------------------------------------------------------| + | 012345 13 01 01 14 PM 01:14:15 PM 15 15 13:14:15 | + +----------------------------------------------------------------------------+ TIME_TO_SEC @@ -1743,11 +1743,11 @@ Example:: os> source=people | eval `TIME_TO_SEC(TIME('22:23:00'))` = TIME_TO_SEC(TIME('22:23:00')) | fields `TIME_TO_SEC(TIME('22:23:00'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | TIME_TO_SEC(TIME('22:23:00')) | - |---------------------------------| - | 80580 | - +---------------------------------+ + +-------------------------------+ + | TIME_TO_SEC(TIME('22:23:00')) | + |-------------------------------| + | 80580 | + +-------------------------------+ TIMEDIFF @@ -1766,11 +1766,11 @@ Example:: os> source=people | eval `TIMEDIFF('23:59:59', '13:00:00')` = TIMEDIFF('23:59:59', '13:00:00') | fields `TIMEDIFF('23:59:59', '13:00:00')` fetched rows / total rows = 1/1 - +------------------------------------+ - | TIMEDIFF('23:59:59', '13:00:00') | - |------------------------------------| - | 10:59:59 | - +------------------------------------+ + +----------------------------------+ + | TIMEDIFF('23:59:59', '13:00:00') | + |----------------------------------| + | 10:59:59 | + +----------------------------------+ TIMESTAMP @@ -1794,11 +1794,11 @@ Example:: os> source=people | eval `TIMESTAMP('2020-08-26 13:49:00')` = TIMESTAMP('2020-08-26 13:49:00'), `TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42'))` = TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | fields `TIMESTAMP('2020-08-26 13:49:00')`, `TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42'))` fetched rows / total rows = 1/1 - +------------------------------------+------------------------------------------------------+ - | TIMESTAMP('2020-08-26 13:49:00') | TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | - |------------------------------------+------------------------------------------------------| - | 2020-08-26 13:49:00 | 2020-08-27 02:04:42 | - +------------------------------------+------------------------------------------------------+ + +----------------------------------+----------------------------------------------------+ + | TIMESTAMP('2020-08-26 13:49:00') | TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | + |----------------------------------+----------------------------------------------------| + | 2020-08-26 13:49:00 | 2020-08-27 02:04:42 | + +----------------------------------+----------------------------------------------------+ TIMESTAMPADD @@ -1819,11 +1819,11 @@ Examples:: os> source=people | eval `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | eval `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` = TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | fields `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')`, `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` fetched rows / total rows = 1/1 - +------------------------------------------------+----------------------------------------------------+ - | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | - |------------------------------------------------+----------------------------------------------------| - | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | - +------------------------------------------------+----------------------------------------------------+ + +----------------------------------------------+--------------------------------------------------+ + | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | + |----------------------------------------------+--------------------------------------------------| + | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | + +----------------------------------------------+--------------------------------------------------+ TIMESTAMPDIFF @@ -1845,11 +1845,11 @@ Examples:: os> source=people | eval `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | eval `TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00'))` = TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | fields `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')`, `TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00'))` fetched rows / total rows = 1/1 - +---------------------------------------------------------------------+-------------------------------------------------------------+ - | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | - |---------------------------------------------------------------------+-------------------------------------------------------------| - | 4 | -23 | - +---------------------------------------------------------------------+-------------------------------------------------------------+ + +-------------------------------------------------------------------+-----------------------------------------------------------+ + | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | + |-------------------------------------------------------------------+-----------------------------------------------------------| + | 4 | -23 | + +-------------------------------------------------------------------+-----------------------------------------------------------+ TO_DAYS @@ -1868,11 +1868,11 @@ Example:: os> source=people | eval `TO_DAYS(DATE('2008-10-07'))` = TO_DAYS(DATE('2008-10-07')) | fields `TO_DAYS(DATE('2008-10-07'))` fetched rows / total rows = 1/1 - +-------------------------------+ - | TO_DAYS(DATE('2008-10-07')) | - |-------------------------------| - | 733687 | - +-------------------------------+ + +-----------------------------+ + | TO_DAYS(DATE('2008-10-07')) | + |-----------------------------| + | 733687 | + +-----------------------------+ TO_SECONDS @@ -1892,11 +1892,11 @@ Example:: os> source=people | eval `TO_SECONDS(DATE('2008-10-07'))` = TO_SECONDS(DATE('2008-10-07')) | eval `TO_SECONDS(950228)` = TO_SECONDS(950228) | fields `TO_SECONDS(DATE('2008-10-07'))`, `TO_SECONDS(950228)` fetched rows / total rows = 1/1 - +----------------------------------+----------------------+ - | TO_SECONDS(DATE('2008-10-07')) | TO_SECONDS(950228) | - |----------------------------------+----------------------| - | 63390556800 | 62961148800 | - +----------------------------------+----------------------+ + +--------------------------------+--------------------+ + | TO_SECONDS(DATE('2008-10-07')) | TO_SECONDS(950228) | + |--------------------------------+--------------------| + | 63390556800 | 62961148800 | + +--------------------------------+--------------------+ UNIX_TIMESTAMP @@ -1918,11 +1918,11 @@ Example:: os> source=people | eval `UNIX_TIMESTAMP(double)` = UNIX_TIMESTAMP(20771122143845), `UNIX_TIMESTAMP(timestamp)` = UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | fields `UNIX_TIMESTAMP(double)`, `UNIX_TIMESTAMP(timestamp)` fetched rows / total rows = 1/1 - +--------------------------+-----------------------------+ - | UNIX_TIMESTAMP(double) | UNIX_TIMESTAMP(timestamp) | - |--------------------------+-----------------------------| - | 3404817525.0 | 848077542.0 | - +--------------------------+-----------------------------+ + +------------------------+---------------------------+ + | UNIX_TIMESTAMP(double) | UNIX_TIMESTAMP(timestamp) | + |------------------------+---------------------------| + | 3404817525.0 | 848077542.0 | + +------------------------+---------------------------+ UTC_DATE @@ -1941,11 +1941,11 @@ Example:: > source=people | eval `UTC_DATE()` = UTC_DATE() | fields `UTC_DATE()` fetched rows / total rows = 1/1 - +--------------+ - | UTC_DATE() | - |--------------| - | 2022-10-03 | - +--------------+ + +------------+ + | UTC_DATE() | + |------------| + | 2022-10-03 | + +------------+ UTC_TIME @@ -1964,11 +1964,11 @@ Example:: > source=people | eval `UTC_TIME()` = UTC_TIME() | fields `UTC_TIME()` fetched rows / total rows = 1/1 - +--------------+ - | UTC_TIME() | - |--------------| - | 17:54:27 | - +--------------+ + +------------+ + | UTC_TIME() | + |------------| + | 17:54:27 | + +------------+ UTC_TIMESTAMP @@ -2053,11 +2053,11 @@ Example:: os> source=people | eval `WEEK(DATE('2008-02-20'))` = WEEK(DATE('2008-02-20')), `WEEK(DATE('2008-02-20'), 1)` = WEEK(DATE('2008-02-20'), 1) | fields `WEEK(DATE('2008-02-20'))`, `WEEK(DATE('2008-02-20'), 1)` fetched rows / total rows = 1/1 - +----------------------------+-------------------------------+ - | WEEK(DATE('2008-02-20')) | WEEK(DATE('2008-02-20'), 1) | - |----------------------------+-------------------------------| - | 7 | 8 | - +----------------------------+-------------------------------+ + +--------------------------+-----------------------------+ + | WEEK(DATE('2008-02-20')) | WEEK(DATE('2008-02-20'), 1) | + |--------------------------+-----------------------------| + | 7 | 8 | + +--------------------------+-----------------------------+ WEEKDAY @@ -2078,11 +2078,11 @@ Example:: os> source=people | eval `weekday(DATE('2020-08-26'))` = weekday(DATE('2020-08-26')) | eval `weekday(DATE('2020-08-27'))` = weekday(DATE('2020-08-27')) | fields `weekday(DATE('2020-08-26'))`, `weekday(DATE('2020-08-27'))` fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+ - | weekday(DATE('2020-08-26')) | weekday(DATE('2020-08-27')) | - |-------------------------------+-------------------------------| - | 2 | 3 | - +-------------------------------+-------------------------------+ + +-----------------------------+-----------------------------+ + | weekday(DATE('2020-08-26')) | weekday(DATE('2020-08-27')) | + |-----------------------------+-----------------------------| + | 2 | 3 | + +-----------------------------+-----------------------------+ WEEK_OF_YEAR @@ -2144,11 +2144,11 @@ Example:: os> source=people | eval `WEEK_OF_YEAR(DATE('2008-02-20'))` = WEEK(DATE('2008-02-20')), `WEEK_OF_YEAR(DATE('2008-02-20'), 1)` = WEEK_OF_YEAR(DATE('2008-02-20'), 1) | fields `WEEK_OF_YEAR(DATE('2008-02-20'))`, `WEEK_OF_YEAR(DATE('2008-02-20'), 1)` fetched rows / total rows = 1/1 - +------------------------------------+---------------------------------------+ - | WEEK_OF_YEAR(DATE('2008-02-20')) | WEEK_OF_YEAR(DATE('2008-02-20'), 1) | - |------------------------------------+---------------------------------------| - | 7 | 8 | - +------------------------------------+---------------------------------------+ + +----------------------------------+-------------------------------------+ + | WEEK_OF_YEAR(DATE('2008-02-20')) | WEEK_OF_YEAR(DATE('2008-02-20'), 1) | + |----------------------------------+-------------------------------------| + | 7 | 8 | + +----------------------------------+-------------------------------------+ YEAR @@ -2167,11 +2167,11 @@ Example:: os> source=people | eval `YEAR(DATE('2020-08-26'))` = YEAR(DATE('2020-08-26')) | fields `YEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +----------------------------+ - | YEAR(DATE('2020-08-26')) | - |----------------------------| - | 2020 | - +----------------------------+ + +--------------------------+ + | YEAR(DATE('2020-08-26')) | + |--------------------------| + | 2020 | + +--------------------------+ YEARWEEK @@ -2190,10 +2190,10 @@ Example:: os> source=people | eval `YEARWEEK('2020-08-26')` = YEARWEEK('2020-08-26') | eval `YEARWEEK('2019-01-05', 1)` = YEARWEEK('2019-01-05', 1) | fields `YEARWEEK('2020-08-26')`, `YEARWEEK('2019-01-05', 1)` fetched rows / total rows = 1/1 - +--------------------------+-----------------------------+ - | YEARWEEK('2020-08-26') | YEARWEEK('2019-01-05', 1) | - |--------------------------+-----------------------------| - | 202034 | 201901 | - +--------------------------+-----------------------------+ + +------------------------+---------------------------+ + | YEARWEEK('2020-08-26') | YEARWEEK('2019-01-05', 1) | + |------------------------+---------------------------| + | 202034 | 201901 | + +------------------------+---------------------------+ diff --git a/docs/user/ppl/functions/expressions.rst b/docs/user/ppl/functions/expressions.rst index ac48324680..d25063d559 100644 --- a/docs/user/ppl/functions/expressions.rst +++ b/docs/user/ppl/functions/expressions.rst @@ -48,13 +48,13 @@ Here is an example for different type of arithmetic expressions:: os> source=accounts | where age > (25 + 5) | fields age ; fetched rows / total rows = 3/3 - +-------+ - | age | - |-------| - | 32 | - | 36 | - | 33 | - +-------+ + +-----+ + | age | + |-----| + | 32 | + | 36 | + | 33 | + +-----+ Predicate Operators =================== @@ -108,11 +108,11 @@ Here is an example for comparison operators:: os> source=accounts | where age > 33 | fields age ; fetched rows / total rows = 1/1 - +-------+ - | age | - |-------| - | 36 | - +-------+ + +-----+ + | age | + |-----| + | 36 | + +-----+ IN @@ -122,12 +122,12 @@ IN operator test field in value lists:: os> source=accounts | where age in (32, 33) | fields age ; fetched rows / total rows = 2/2 - +-------+ - | age | - |-------| - | 32 | - | 33 | - +-------+ + +-----+ + | age | + |-----| + | 32 | + | 33 | + +-----+ OR @@ -137,12 +137,12 @@ OR operator :: os> source=accounts | where age = 32 OR age = 33 | fields age ; fetched rows / total rows = 2/2 - +-------+ - | age | - |-------| - | 32 | - | 33 | - +-------+ + +-----+ + | age | + |-----| + | 32 | + | 33 | + +-----+ NOT @@ -152,10 +152,10 @@ NOT operator :: os> source=accounts | where not age in (32, 33) | fields age ; fetched rows / total rows = 2/2 - +-------+ - | age | - |-------| - | 36 | - | 28 | - +-------+ + +-----+ + | age | + |-----| + | 36 | + | 28 | + +-----+ diff --git a/docs/user/ppl/functions/math.rst b/docs/user/ppl/functions/math.rst index c5eb07b5da..65f544461b 100644 --- a/docs/user/ppl/functions/math.rst +++ b/docs/user/ppl/functions/math.rst @@ -25,11 +25,11 @@ Example:: os> source=people | eval `ABS(-1)` = ABS(-1) | fields `ABS(-1)` fetched rows / total rows = 1/1 - +-----------+ - | ABS(-1) | - |-----------| - | 1 | - +-----------+ + +---------+ + | ABS(-1) | + |---------| + | 1 | + +---------+ ACOS @@ -71,11 +71,11 @@ Example:: os> source=people | eval `ASIN(0)` = ASIN(0) | fields `ASIN(0)` fetched rows / total rows = 1/1 - +-----------+ - | ASIN(0) | - |-----------| - | 0.0 | - +-----------+ + +---------+ + | ASIN(0) | + |---------| + | 0.0 | + +---------+ ATAN @@ -150,19 +150,19 @@ Example:: os> source=people | eval `CEILING(0)` = CEILING(0), `CEILING(50.00005)` = CEILING(50.00005), `CEILING(-50.00005)` = CEILING(-50.00005) | fields `CEILING(0)`, `CEILING(50.00005)`, `CEILING(-50.00005)` fetched rows / total rows = 1/1 - +--------------+---------------------+----------------------+ - | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | - |--------------+---------------------+----------------------| - | 0 | 51 | -50 | - +--------------+---------------------+----------------------+ + +------------+-------------------+--------------------+ + | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | + |------------+-------------------+--------------------| + | 0 | 51 | -50 | + +------------+-------------------+--------------------+ os> source=people | eval `CEILING(3147483647.12345)` = CEILING(3147483647.12345), `CEILING(113147483647.12345)` = CEILING(113147483647.12345), `CEILING(3147483647.00001)` = CEILING(3147483647.00001) | fields `CEILING(3147483647.12345)`, `CEILING(113147483647.12345)`, `CEILING(3147483647.00001)` fetched rows / total rows = 1/1 - +-----------------------------+-------------------------------+-----------------------------+ - | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | - |-----------------------------+-------------------------------+-----------------------------| - | 3147483648 | 113147483648 | 3147483648 | - +-----------------------------+-------------------------------+-----------------------------+ + +---------------------------+-----------------------------+---------------------------+ + | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | + |---------------------------+-----------------------------+---------------------------| + | 3147483648 | 113147483648 | 3147483648 | + +---------------------------+-----------------------------+---------------------------+ CONV @@ -181,11 +181,11 @@ Example:: os> source=people | eval `CONV('12', 10, 16)` = CONV('12', 10, 16), `CONV('2C', 16, 10)` = CONV('2C', 16, 10), `CONV(12, 10, 2)` = CONV(12, 10, 2), `CONV(1111, 2, 10)` = CONV(1111, 2, 10) | fields `CONV('12', 10, 16)`, `CONV('2C', 16, 10)`, `CONV(12, 10, 2)`, `CONV(1111, 2, 10)` fetched rows / total rows = 1/1 - +----------------------+----------------------+-------------------+---------------------+ - | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | - |----------------------+----------------------+-------------------+---------------------| - | c | 44 | 1100 | 15 | - +----------------------+----------------------+-------------------+---------------------+ + +--------------------+--------------------+-----------------+-------------------+ + | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | + |--------------------+--------------------+-----------------+-------------------| + | c | 44 | 1100 | 15 | + +--------------------+--------------------+-----------------+-------------------+ COS @@ -204,11 +204,11 @@ Example:: os> source=people | eval `COS(0)` = COS(0) | fields `COS(0)` fetched rows / total rows = 1/1 - +----------+ - | COS(0) | - |----------| - | 1.0 | - +----------+ + +--------+ + | COS(0) | + |--------| + | 1.0 | + +--------+ COT @@ -250,11 +250,11 @@ Example:: os> source=people | eval `CRC32('MySQL')` = CRC32('MySQL') | fields `CRC32('MySQL')` fetched rows / total rows = 1/1 - +------------------+ - | CRC32('MySQL') | - |------------------| - | 3259397556 | - +------------------+ + +----------------+ + | CRC32('MySQL') | + |----------------| + | 3259397556 | + +----------------+ DEGREES @@ -342,27 +342,27 @@ Example:: os> source=people | eval `FLOOR(0)` = FLOOR(0), `FLOOR(50.00005)` = FLOOR(50.00005), `FLOOR(-50.00005)` = FLOOR(-50.00005) | fields `FLOOR(0)`, `FLOOR(50.00005)`, `FLOOR(-50.00005)` fetched rows / total rows = 1/1 - +------------+-------------------+--------------------+ - | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | - |------------+-------------------+--------------------| - | 0 | 50 | -51 | - +------------+-------------------+--------------------+ + +----------+-----------------+------------------+ + | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | + |----------+-----------------+------------------| + | 0 | 50 | -51 | + +----------+-----------------+------------------+ os> source=people | eval `FLOOR(3147483647.12345)` = FLOOR(3147483647.12345), `FLOOR(113147483647.12345)` = FLOOR(113147483647.12345), `FLOOR(3147483647.00001)` = FLOOR(3147483647.00001) | fields `FLOOR(3147483647.12345)`, `FLOOR(113147483647.12345)`, `FLOOR(3147483647.00001)` fetched rows / total rows = 1/1 - +---------------------------+-----------------------------+---------------------------+ - | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | - |---------------------------+-----------------------------+---------------------------| - | 3147483647 | 113147483647 | 3147483647 | - +---------------------------+-----------------------------+---------------------------+ + +-------------------------+---------------------------+-------------------------+ + | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | + |-------------------------+---------------------------+-------------------------| + | 3147483647 | 113147483647 | 3147483647 | + +-------------------------+---------------------------+-------------------------+ os> source=people | eval `FLOOR(282474973688888.022)` = FLOOR(282474973688888.022), `FLOOR(9223372036854775807.022)` = FLOOR(9223372036854775807.022), `FLOOR(9223372036854775807.0000001)` = FLOOR(9223372036854775807.0000001) | fields `FLOOR(282474973688888.022)`, `FLOOR(9223372036854775807.022)`, `FLOOR(9223372036854775807.0000001)` fetched rows / total rows = 1/1 - +------------------------------+----------------------------------+--------------------------------------+ - | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | - |------------------------------+----------------------------------+--------------------------------------| - | 282474973688888 | 9223372036854775807 | 9223372036854775807 | - +------------------------------+----------------------------------+--------------------------------------+ + +----------------------------+--------------------------------+------------------------------------+ + | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | + |----------------------------+--------------------------------+------------------------------------| + | 282474973688888 | 9223372036854775807 | 9223372036854775807 | + +----------------------------+--------------------------------+------------------------------------+ LN @@ -406,11 +406,11 @@ Example:: os> source=people | eval `LOG(2)` = LOG(2), `LOG(2, 8)` = LOG(2, 8) | fields `LOG(2)`, `LOG(2, 8)` fetched rows / total rows = 1/1 - +--------------------+-------------+ - | LOG(2) | LOG(2, 8) | - |--------------------+-------------| - | 0.6931471805599453 | 3.0 | - +--------------------+-------------+ + +--------------------+-----------+ + | LOG(2) | LOG(2, 8) | + |--------------------+-----------| + | 0.6931471805599453 | 3.0 | + +--------------------+-----------+ LOG2 @@ -431,11 +431,11 @@ Example:: os> source=people | eval `LOG2(8)` = LOG2(8) | fields `LOG2(8)` fetched rows / total rows = 1/1 - +-----------+ - | LOG2(8) | - |-----------| - | 3.0 | - +-----------+ + +---------+ + | LOG2(8) | + |---------| + | 3.0 | + +---------+ LOG10 @@ -456,11 +456,11 @@ Example:: os> source=people | eval `LOG10(100)` = LOG10(100) | fields `LOG10(100)` fetched rows / total rows = 1/1 - +--------------+ - | LOG10(100) | - |--------------| - | 2.0 | - +--------------+ + +------------+ + | LOG10(100) | + |------------| + | 2.0 | + +------------+ MOD @@ -479,11 +479,11 @@ Example:: os> source=people | eval `MOD(3, 2)` = MOD(3, 2), `MOD(3.1, 2)` = MOD(3.1, 2) | fields `MOD(3, 2)`, `MOD(3.1, 2)` fetched rows / total rows = 1/1 - +-------------+---------------+ - | MOD(3, 2) | MOD(3.1, 2) | - |-------------+---------------| - | 1 | 1.1 | - +-------------+---------------+ + +-----------+-------------+ + | MOD(3, 2) | MOD(3.1, 2) | + |-----------+-------------| + | 1 | 1.1 | + +-----------+-------------+ PI @@ -525,11 +525,11 @@ Example:: os> source=people | eval `POW(3, 2)` = POW(3, 2), `POW(-3, 2)` = POW(-3, 2), `POW(3, -2)` = POW(3, -2) | fields `POW(3, 2)`, `POW(-3, 2)`, `POW(3, -2)` fetched rows / total rows = 1/1 - +-------------+--------------+--------------------+ - | POW(3, 2) | POW(-3, 2) | POW(3, -2) | - |-------------+--------------+--------------------| - | 9.0 | 9.0 | 0.1111111111111111 | - +-------------+--------------+--------------------+ + +-----------+------------+--------------------+ + | POW(3, 2) | POW(-3, 2) | POW(3, -2) | + |-----------+------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +-----------+------------+--------------------+ POWER @@ -550,11 +550,11 @@ Example:: os> source=people | eval `POWER(3, 2)` = POWER(3, 2), `POWER(-3, 2)` = POWER(-3, 2), `POWER(3, -2)` = POWER(3, -2) | fields `POWER(3, 2)`, `POWER(-3, 2)`, `POWER(3, -2)` fetched rows / total rows = 1/1 - +---------------+----------------+--------------------+ - | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | - |---------------+----------------+--------------------| - | 9.0 | 9.0 | 0.1111111111111111 | - +---------------+----------------+--------------------+ + +-------------+--------------+--------------------+ + | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | + |-------------+--------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +-------------+--------------+--------------------+ RADIANS @@ -622,11 +622,11 @@ Example:: os> source=people | eval `ROUND(12.34)` = ROUND(12.34), `ROUND(12.34, 1)` = ROUND(12.34, 1), `ROUND(12.34, -1)` = ROUND(12.34, -1), `ROUND(12, 1)` = ROUND(12, 1) | fields `ROUND(12.34)`, `ROUND(12.34, 1)`, `ROUND(12.34, -1)`, `ROUND(12, 1)` fetched rows / total rows = 1/1 - +----------------+-------------------+--------------------+----------------+ - | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | - |----------------+-------------------+--------------------+----------------| - | 12.0 | 12.3 | 10.0 | 12 | - +----------------+-------------------+--------------------+----------------+ + +--------------+-----------------+------------------+--------------+ + | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | + |--------------+-----------------+------------------+--------------| + | 12.0 | 12.3 | 10.0 | 12 | + +--------------+-----------------+------------------+--------------+ SIGN @@ -645,11 +645,11 @@ Example:: os> source=people | eval `SIGN(1)` = SIGN(1), `SIGN(0)` = SIGN(0), `SIGN(-1.1)` = SIGN(-1.1) | fields `SIGN(1)`, `SIGN(0)`, `SIGN(-1.1)` fetched rows / total rows = 1/1 - +-----------+-----------+--------------+ - | SIGN(1) | SIGN(0) | SIGN(-1.1) | - |-----------+-----------+--------------| - | 1 | 0 | -1 | - +-----------+-----------+--------------+ + +---------+---------+------------+ + | SIGN(1) | SIGN(0) | SIGN(-1.1) | + |---------+---------+------------| + | 1 | 0 | -1 | + +---------+---------+------------+ SIN @@ -668,11 +668,11 @@ Example:: os> source=people | eval `SIN(0)` = SIN(0) | fields `SIN(0)` fetched rows / total rows = 1/1 - +----------+ - | SIN(0) | - |----------| - | 0.0 | - +----------+ + +--------+ + | SIN(0) | + |--------| + | 0.0 | + +--------+ SQRT @@ -694,11 +694,11 @@ Example:: os> source=people | eval `SQRT(4)` = SQRT(4), `SQRT(4.41)` = SQRT(4.41) | fields `SQRT(4)`, `SQRT(4.41)` fetched rows / total rows = 1/1 - +-----------+--------------+ - | SQRT(4) | SQRT(4.41) | - |-----------+--------------| - | 2.0 | 2.1 | - +-----------+--------------+ + +---------+------------+ + | SQRT(4) | SQRT(4.41) | + |---------+------------| + | 2.0 | 2.1 | + +---------+------------+ CBRT @@ -719,9 +719,9 @@ Example:: opensearchsql> source=location | eval `CBRT(8)` = CBRT(8), `CBRT(9.261)` = CBRT(9.261), `CBRT(-27)` = CBRT(-27) | fields `CBRT(8)`, `CBRT(9.261)`, `CBRT(-27)`; fetched rows / total rows = 2/2 - +-----------+---------------+-------------+ - | CBRT(8) | CBRT(9.261) | CBRT(-27) | - |-----------+---------------+-------------| - | 2.0 | 2.1 | -3.0 | - | 2.0 | 2.1 | -3.0 | - +-----------+---------------+-------------+ + +---------+-------------+-----------+ + | CBRT(8) | CBRT(9.261) | CBRT(-27) | + |---------+-------------+-----------| + | 2.0 | 2.1 | -3.0 | + | 2.0 | 2.1 | -3.0 | + +---------+-------------+-----------+ diff --git a/docs/user/ppl/functions/relevance.rst b/docs/user/ppl/functions/relevance.rst index fb31edb0d2..a1f240ee05 100644 --- a/docs/user/ppl/functions/relevance.rst +++ b/docs/user/ppl/functions/relevance.rst @@ -37,12 +37,12 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> source=accounts | where match(address, 'Street') | fields lastname, address; fetched rows / total rows = 2/2 - +------------+--------------------+ - | lastname | address | - |------------+--------------------| - | Bond | 671 Bristol Street | - | Bates | 789 Madison Street | - +------------+--------------------+ + +----------+--------------------+ + | lastname | address | + |----------+--------------------| + | Bond | 671 Bristol Street | + | Bates | 789 Madison Street | + +----------+--------------------+ @@ -50,11 +50,11 @@ Another example to show how to set custom values for the optional parameters:: os> source=accounts | where match(firstname, 'Hattie', operator='AND', boost=2.0) | fields lastname; fetched rows / total rows = 1/1 - +------------+ - | lastname | - |------------| - | Bond | - +------------+ + +----------+ + | lastname | + |----------| + | Bond | + +----------+ MATCH_PHRASE @@ -175,22 +175,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> source=books | where multi_match(['title'], 'Pooh House') | fields id, title, author; fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> source=books | where multi_match(['title'], 'Pooh House', operator='AND', analyzer=default) | fields id, title, author; fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ SIMPLE_QUERY_STRING @@ -228,22 +228,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> source=books | where simple_query_string(['title'], 'Pooh House') | fields id, title, author; fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> source=books | where simple_query_string(['title'], 'Pooh House', flags='ALL', default_operator='AND') | fields id, title, author; fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ MATCH_BOOL_PREFIX @@ -270,22 +270,22 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> source=accounts | where match_bool_prefix(address, 'Bristol Stre') | fields firstname, address fetched rows / total rows = 2/2 - +-------------+--------------------+ - | firstname | address | - |-------------+--------------------| - | Hattie | 671 Bristol Street | - | Nanette | 789 Madison Street | - +-------------+--------------------+ + +-----------+--------------------+ + | firstname | address | + |-----------+--------------------| + | Hattie | 671 Bristol Street | + | Nanette | 789 Madison Street | + +-----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> source=accounts | where match_bool_prefix(address, 'Bristol Stre', minimum_should_match = 2) | fields firstname, address fetched rows / total rows = 1/1 - +-------------+--------------------+ - | firstname | address | - |-------------+--------------------| - | Hattie | 671 Bristol Street | - +-------------+--------------------+ + +-----------+--------------------+ + | firstname | address | + |-----------+--------------------| + | Hattie | 671 Bristol Street | + +-----------+--------------------+ QUERY_STRING @@ -335,22 +335,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> source=books | where query_string(['title'], 'Pooh House') | fields id, title, author; fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> source=books | where query_string(['title'], 'Pooh House', default_operator='AND') | fields id, title, author; fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ Limitations >>>>>>>>>>> diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index edf5220f4f..0dbb09cbb8 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -24,11 +24,11 @@ Example:: os> source=people | eval `CONCAT('hello', 'world')` = CONCAT('hello', 'world'), `CONCAT('hello ', 'whole ', 'world', '!')` = CONCAT('hello ', 'whole ', 'world', '!') | fields `CONCAT('hello', 'world')`, `CONCAT('hello ', 'whole ', 'world', '!')` fetched rows / total rows = 1/1 - +----------------------------+--------------------------------------------+ - | CONCAT('hello', 'world') | CONCAT('hello ', 'whole ', 'world', '!') | - |----------------------------+--------------------------------------------| - | helloworld | hello whole world! | - +----------------------------+--------------------------------------------+ + +--------------------------+------------------------------------------+ + | CONCAT('hello', 'world') | CONCAT('hello ', 'whole ', 'world', '!') | + |--------------------------+------------------------------------------| + | helloworld | hello whole world! | + +--------------------------+------------------------------------------+ CONCAT_WS @@ -47,11 +47,11 @@ Example:: os> source=people | eval `CONCAT_WS(',', 'hello', 'world')` = CONCAT_WS(',', 'hello', 'world') | fields `CONCAT_WS(',', 'hello', 'world')` fetched rows / total rows = 1/1 - +------------------------------------+ - | CONCAT_WS(',', 'hello', 'world') | - |------------------------------------| - | hello,world | - +------------------------------------+ + +----------------------------------+ + | CONCAT_WS(',', 'hello', 'world') | + |----------------------------------| + | hello,world | + +----------------------------------+ LENGTH @@ -74,11 +74,11 @@ Example:: os> source=people | eval `LENGTH('helloworld')` = LENGTH('helloworld') | fields `LENGTH('helloworld')` fetched rows / total rows = 1/1 - +------------------------+ - | LENGTH('helloworld') | - |------------------------| - | 10 | - +------------------------+ + +----------------------+ + | LENGTH('helloworld') | + |----------------------| + | 10 | + +----------------------+ LIKE @@ -98,11 +98,11 @@ Example:: os> source=people | eval `LIKE('hello world', '_ello%')` = LIKE('hello world', '_ELLO%') | fields `LIKE('hello world', '_ello%')` fetched rows / total rows = 1/1 - +---------------------------------+ - | LIKE('hello world', '_ello%') | - |---------------------------------| - | True | - +---------------------------------+ + +-------------------------------+ + | LIKE('hello world', '_ello%') | + |-------------------------------| + | True | + +-------------------------------+ LOWER @@ -121,11 +121,11 @@ Example:: os> source=people | eval `LOWER('helloworld')` = LOWER('helloworld'), `LOWER('HELLOWORLD')` = LOWER('HELLOWORLD') | fields `LOWER('helloworld')`, `LOWER('HELLOWORLD')` fetched rows / total rows = 1/1 - +-----------------------+-----------------------+ - | LOWER('helloworld') | LOWER('HELLOWORLD') | - |-----------------------+-----------------------| - | helloworld | helloworld | - +-----------------------+-----------------------+ + +---------------------+---------------------+ + | LOWER('helloworld') | LOWER('HELLOWORLD') | + |---------------------+---------------------| + | helloworld | helloworld | + +---------------------+---------------------+ LTRIM @@ -144,11 +144,11 @@ Example:: os> source=people | eval `LTRIM(' hello')` = LTRIM(' hello'), `LTRIM('hello ')` = LTRIM('hello ') | fields `LTRIM(' hello')`, `LTRIM('hello ')` fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | LTRIM(' hello') | LTRIM('hello ') | - |---------------------+---------------------| - | hello | hello | - +---------------------+---------------------+ + +-------------------+-------------------+ + | LTRIM(' hello') | LTRIM('hello ') | + |-------------------+-------------------| + | hello | hello | + +-------------------+-------------------+ POSITION @@ -169,11 +169,11 @@ Example:: os> source=people | eval `POSITION('world' IN 'helloworld')` = POSITION('world' IN 'helloworld'), `POSITION('invalid' IN 'helloworld')`= POSITION('invalid' IN 'helloworld') | fields `POSITION('world' IN 'helloworld')`, `POSITION('invalid' IN 'helloworld')` fetched rows / total rows = 1/1 - +-------------------------------------+---------------------------------------+ - | POSITION('world' IN 'helloworld') | POSITION('invalid' IN 'helloworld') | - |-------------------------------------+---------------------------------------| - | 6 | 0 | - +-------------------------------------+---------------------------------------+ + +-----------------------------------+-------------------------------------+ + | POSITION('world' IN 'helloworld') | POSITION('invalid' IN 'helloworld') | + |-----------------------------------+-------------------------------------| + | 6 | 0 | + +-----------------------------------+-------------------------------------+ REVERSE @@ -192,11 +192,11 @@ Example:: os> source=people | eval `REVERSE('abcde')` = REVERSE('abcde') | fields `REVERSE('abcde')` fetched rows / total rows = 1/1 - +--------------------+ - | REVERSE('abcde') | - |--------------------| - | edcba | - +--------------------+ + +------------------+ + | REVERSE('abcde') | + |------------------| + | edcba | + +------------------+ RIGHT @@ -215,11 +215,11 @@ Example:: os> source=people | eval `RIGHT('helloworld', 5)` = RIGHT('helloworld', 5), `RIGHT('HELLOWORLD', 0)` = RIGHT('HELLOWORLD', 0) | fields `RIGHT('helloworld', 5)`, `RIGHT('HELLOWORLD', 0)` fetched rows / total rows = 1/1 - +--------------------------+--------------------------+ - | RIGHT('helloworld', 5) | RIGHT('HELLOWORLD', 0) | - |--------------------------+--------------------------| - | world | | - +--------------------------+--------------------------+ + +------------------------+------------------------+ + | RIGHT('helloworld', 5) | RIGHT('HELLOWORLD', 0) | + |------------------------+------------------------| + | world | | + +------------------------+------------------------+ RTRIM @@ -238,11 +238,11 @@ Example:: os> source=people | eval `RTRIM(' hello')` = RTRIM(' hello'), `RTRIM('hello ')` = RTRIM('hello ') | fields `RTRIM(' hello')`, `RTRIM('hello ')` fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | RTRIM(' hello') | RTRIM('hello ') | - |---------------------+---------------------| - | hello | hello | - +---------------------+---------------------+ + +-------------------+-------------------+ + | RTRIM(' hello') | RTRIM('hello ') | + |-------------------+-------------------| + | hello | hello | + +-------------------+-------------------+ SUBSTRING @@ -263,11 +263,11 @@ Example:: os> source=people | eval `SUBSTRING('helloworld', 5)` = SUBSTRING('helloworld', 5), `SUBSTRING('helloworld', 5, 3)` = SUBSTRING('helloworld', 5, 3) | fields `SUBSTRING('helloworld', 5)`, `SUBSTRING('helloworld', 5, 3)` fetched rows / total rows = 1/1 - +------------------------------+---------------------------------+ - | SUBSTRING('helloworld', 5) | SUBSTRING('helloworld', 5, 3) | - |------------------------------+---------------------------------| - | oworld | owo | - +------------------------------+---------------------------------+ + +----------------------------+-------------------------------+ + | SUBSTRING('helloworld', 5) | SUBSTRING('helloworld', 5, 3) | + |----------------------------+-------------------------------| + | oworld | owo | + +----------------------------+-------------------------------+ TRIM @@ -284,11 +284,11 @@ Example:: os> source=people | eval `TRIM(' hello')` = TRIM(' hello'), `TRIM('hello ')` = TRIM('hello ') | fields `TRIM(' hello')`, `TRIM('hello ')` fetched rows / total rows = 1/1 - +--------------------+--------------------+ - | TRIM(' hello') | TRIM('hello ') | - |--------------------+--------------------| - | hello | hello | - +--------------------+--------------------+ + +------------------+------------------+ + | TRIM(' hello') | TRIM('hello ') | + |------------------+------------------| + | hello | hello | + +------------------+------------------+ UPPER @@ -307,8 +307,8 @@ Example:: os> source=people | eval `UPPER('helloworld')` = UPPER('helloworld'), `UPPER('HELLOWORLD')` = UPPER('HELLOWORLD') | fields `UPPER('helloworld')`, `UPPER('HELLOWORLD')` fetched rows / total rows = 1/1 - +-----------------------+-----------------------+ - | UPPER('helloworld') | UPPER('HELLOWORLD') | - |-----------------------+-----------------------| - | HELLOWORLD | HELLOWORLD | - +-----------------------+-----------------------+ + +---------------------+---------------------+ + | UPPER('helloworld') | UPPER('HELLOWORLD') | + |---------------------+---------------------| + | HELLOWORLD | HELLOWORLD | + +---------------------+---------------------+ diff --git a/docs/user/ppl/functions/system.rst b/docs/user/ppl/functions/system.rst index fbe9860dce..698933a3c4 100644 --- a/docs/user/ppl/functions/system.rst +++ b/docs/user/ppl/functions/system.rst @@ -24,8 +24,8 @@ Example:: os> source=people | eval `typeof(date)` = typeof(DATE('2008-04-14')), `typeof(int)` = typeof(1), `typeof(now())` = typeof(now()), `typeof(column)` = typeof(accounts) | fields `typeof(date)`, `typeof(int)`, `typeof(now())`, `typeof(column)` fetched rows / total rows = 1/1 - +----------------+---------------+-----------------+------------------+ - | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | - |----------------+---------------+-----------------+------------------| - | DATE | INTEGER | DATETIME | OBJECT | - +----------------+---------------+-----------------+------------------+ + +--------------+-------------+---------------+----------------+ + | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | + |--------------+-------------+---------------+----------------| + | DATE | INTEGER | TIMESTAMP | OBJECT | + +--------------+-------------+---------------+----------------+ diff --git a/docs/user/ppl/general/datatypes.rst b/docs/user/ppl/general/datatypes.rst index cabc689526..cd2e6d35e3 100644 --- a/docs/user/ppl/general/datatypes.rst +++ b/docs/user/ppl/general/datatypes.rst @@ -383,11 +383,11 @@ PPL query:: os> source=people | fields city, city.name, city.location.latitude; fetched rows / total rows = 1/1 - +-----------------------------------------------------+-------------+--------------------------+ - | city | city.name | city.location.latitude | - |-----------------------------------------------------+-------------+--------------------------| - | {'name': 'Seattle', 'location': {'latitude': 10.5}} | Seattle | 10.5 | - +-----------------------------------------------------+-------------+--------------------------+ + +-----------------------------------------------------+-----------+------------------------+ + | city | city.name | city.location.latitude | + |-----------------------------------------------------+-----------+------------------------| + | {'name': 'Seattle', 'location': {'latitude': 10.5}} | Seattle | 10.5 | + +-----------------------------------------------------+-----------+------------------------+ Example 2: Group by struct inner attribute @@ -399,11 +399,11 @@ PPL query:: os> source=people | stats count() by city.name; fetched rows / total rows = 1/1 - +-----------+-------------+ - | count() | city.name | - |-----------+-------------| - | 1 | Seattle | - +-----------+-------------+ + +---------+-----------+ + | count() | city.name | + |---------+-----------| + | 1 | Seattle | + +---------+-----------+ Example 3: Selecting Field of Array Value ----------------------------------------- @@ -412,8 +412,8 @@ Select deeper level for object fields of array value which returns the first ele os> source = people | fields accounts, accounts.id; fetched rows / total rows = 1/1 - +------------+---------------+ - | accounts | accounts.id | - |------------+---------------| - | {'id': 1} | 1 | - +------------+---------------+ \ No newline at end of file + +-----------+-------------+ + | accounts | accounts.id | + |-----------+-------------| + | {'id': 1} | 1 | + +-----------+-------------+ \ No newline at end of file diff --git a/docs/user/ppl/general/identifiers.rst b/docs/user/ppl/general/identifiers.rst index 51fc36c40f..bab540ffdd 100644 --- a/docs/user/ppl/general/identifiers.rst +++ b/docs/user/ppl/general/identifiers.rst @@ -39,14 +39,14 @@ Here are examples for using index pattern directly without quotes:: os> source=accounts | fields account_number, firstname, lastname; fetched rows / total rows = 4/4 - +------------------+-------------+------------+ - | account_number | firstname | lastname | - |------------------+-------------+------------| - | 1 | Amber | Duke | - | 6 | Hattie | Bond | - | 13 | Nanette | Bates | - | 18 | Dale | Adams | - +------------------+-------------+------------+ + +----------------+-----------+----------+ + | account_number | firstname | lastname | + |----------------+-----------+----------| + | 1 | Amber | Duke | + | 6 | Hattie | Bond | + | 13 | Nanette | Bates | + | 18 | Dale | Adams | + +----------------+-----------+----------+ Delimited Identifiers @@ -73,14 +73,14 @@ Here are examples for quoting an index name by back ticks:: os> source=`accounts` | fields `account_number`; fetched rows / total rows = 4/4 - +------------------+ - | account_number | - |------------------| - | 1 | - | 6 | - | 13 | - | 18 | - +------------------+ + +----------------+ + | account_number | + |----------------| + | 1 | + | 6 | + | 13 | + | 18 | + +----------------+ Cross-Cluster Index Identifiers @@ -135,29 +135,29 @@ Query wildcard indices:: os> source=acc* | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 5 | - +-----------+ + +---------+ + | count() | + |---------| + | 5 | + +---------+ Query multiple indices seperated by ``,``:: os> source=accounts, account2 | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 5 | - +-----------+ + +---------+ + | count() | + |---------| + | 5 | + +---------+ Query delimited multiple indices seperated by ``,``:: os> source=`accounts,account2` | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 5 | - +-----------+ + +---------+ + | count() | + |---------| + | 5 | + +---------+ diff --git a/doctest/build.gradle b/doctest/build.gradle index 564362df23..4fe20ba212 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -21,8 +21,16 @@ def path = project(':').projectDir def plugin_path = project(':doctest').projectDir task cloneSqlCli(type: Exec) { - // clone the sql-cli repo locally - commandLine 'git', 'clone', 'https://github.com/opensearch-project/sql-cli.git' + def repoDir = new File("${project.projectDir}/sql-cli") + + if (repoDir.exists()) { + // Repository already exists, fetch and checkout latest + commandLine 'git', '-C', repoDir.absolutePath, 'fetch', 'origin', 'main' + commandLine 'git', '-C', repoDir.absolutePath, 'checkout', 'origin/main' + } else { + // Repository doesn't exist, clone it + commandLine 'git', 'clone', 'https://github.com/opensearch-project/sql-cli.git', repoDir.absolutePath + } } task bootstrap(type: Exec, dependsOn: ['cloneSqlCli']) { diff --git a/doctest/test_docs.py b/doctest/test_docs.py index 1fedbdf49e..881078a9bd 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -48,10 +48,34 @@ def process(self, statement): click.echo(output) +""" +For _explain requests, there are several additional request fields that will inconsistently +appear/change depending on underlying cluster state. This method normalizes these responses in-place +to make _explain doctests more consistent. + +If the passed response is not an _explain response, the input is left unmodified. +""" +def normalize_explain_response(data): + if "root" in data: + data = data["root"] + + if (request := data.get("description", {}).get("request", None)) and request.startswith("OpenSearchQueryRequest("): + for filter_field in ["needClean", "pitId", "cursorKeepAlive", "searchAfter", "searchResponse"]: + request = re.sub(f", {filter_field}=\\w+", "", request) + data["description"]["request"] = request + + for child in data.get("children", []): + normalize_explain_response(child) + + return data + + def pretty_print(s): try: - d = json.loads(s) - print(json.dumps(d, indent=2)) + data = json.loads(s) + normalize_explain_response(data) + + print(json.dumps(data, indent=2)) except json.decoder.JSONDecodeError: print(s) From 81577dfc8035a1baf0751e87e3ac98736021c5dd Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 24 Oct 2024 20:09:56 -0700 Subject: [PATCH 26/32] Array values are preserved (#1300) (#3095) (#3120) Signed-off-by: Norman Jordan (cherry picked from commit e109417bb3ce770fb54cfd7b231b8219a22ed46f) Co-authored-by: normanj-bitquill <78755797+normanj-bitquill@users.noreply.github.com> --- .../sql/common/setting/Settings.java | 3 + docs/user/beyond/partiql.rst | 10 +- docs/user/ppl/general/datatypes.rst | 10 +- .../sql/legacy/ObjectFieldSelectIT.java | 16 +- .../org/opensearch/sql/ppl/StandaloneIT.java | 1 + .../java/org/opensearch/sql/sql/NestedIT.java | 2 +- .../sql/sql/StandalonePaginationIT.java | 1 + .../value/OpenSearchExprValueFactory.java | 9 +- .../request/OpenSearchQueryRequest.java | 4 +- .../request/OpenSearchScrollRequest.java | 4 +- .../setting/OpenSearchSettings.java | 18 +- .../opensearch/storage/OpenSearchIndex.java | 7 +- .../storage/scan/OpenSearchIndexScan.java | 2 - .../storage/script/core/ExpressionScript.java | 2 +- .../client/OpenSearchNodeClientTest.java | 1 - .../client/OpenSearchRestClientTest.java | 1 - .../value/OpenSearchExprValueFactoryTest.java | 220 +++++++++++++++--- .../request/OpenSearchQueryRequestTest.java | 3 +- .../request/OpenSearchRequestBuilderTest.java | 1 + .../response/OpenSearchResponseTest.java | 16 +- .../storage/OpenSearchIndexTest.java | 11 + .../OpenSearchIndexScanPaginationTest.java | 4 +- .../storage/scan/OpenSearchIndexScanTest.java | 4 +- 23 files changed, 284 insertions(+), 66 deletions(-) diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 0037032d22..a9fa693a22 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -28,6 +28,9 @@ public enum Key { /** PPL Settings. */ PPL_ENABLED("plugins.ppl.enabled"), + /** Query Settings. */ + FIELD_TYPE_TOLERANCE("plugins.query.field_type_tolerance"), + /** Common Settings for SQL and PPL. */ QUERY_MEMORY_LIMIT("plugins.query.memory_limit"), QUERY_SIZE_LIMIT("plugins.query.size_limit"), diff --git a/docs/user/beyond/partiql.rst b/docs/user/beyond/partiql.rst index 6a93a55c94..d8e4b0722b 100644 --- a/docs/user/beyond/partiql.rst +++ b/docs/user/beyond/partiql.rst @@ -202,11 +202,11 @@ Selecting top level for object fields, object fields of array value and nested f os> SELECT city, accounts, projects FROM people; fetched rows / total rows = 1/1 - +-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------+ - | city | accounts | projects | - |-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------| - | {'name': 'Seattle', 'location': {'latitude': 10.5}} | {'id': 1} | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] | - +-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------+ + +-----------------------------------------------------+-----------------------+----------------------------------------------------------------------------------------------------------------+ + | city | accounts | projects | + |-----------------------------------------------------+-----------------------+----------------------------------------------------------------------------------------------------------------| + | {'name': 'Seattle', 'location': {'latitude': 10.5}} | [{'id': 1},{'id': 2}] | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] | + +-----------------------------------------------------+-----------------------+----------------------------------------------------------------------------------------------------------------+ Example 2: Selecting Deeper Levels ---------------------------------- diff --git a/docs/user/ppl/general/datatypes.rst b/docs/user/ppl/general/datatypes.rst index cd2e6d35e3..27062f308f 100644 --- a/docs/user/ppl/general/datatypes.rst +++ b/docs/user/ppl/general/datatypes.rst @@ -412,8 +412,8 @@ Select deeper level for object fields of array value which returns the first ele os> source = people | fields accounts, accounts.id; fetched rows / total rows = 1/1 - +-----------+-------------+ - | accounts | accounts.id | - |-----------+-------------| - | {'id': 1} | 1 | - +-----------+-------------+ \ No newline at end of file + +-----------------------+-------------+ + | accounts | accounts.id | + |-----------------------+-------------| + | [{'id': 1},{'id': 2}] | 1 | + +-----------------------+-------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java index 3a2f48d497..aadd79469d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java @@ -14,6 +14,7 @@ import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.utils.StringUtils; /** @@ -79,9 +80,20 @@ public void testSelectNestedFieldItself() { @Test public void testSelectObjectFieldOfArrayValuesItself() { JSONObject response = new JSONObject(query("SELECT accounts FROM %s")); + verifyDataRows(response, rows(new JSONArray("[{\"id\":1},{\"id\":2}]"))); + } - // Only the first element of the list of is returned. - verifyDataRows(response, rows(new JSONObject("{\"id\": 1}"))); + @Test + public void testSelectObjectFieldOfArrayValuesItselfNoFieldTypeTolerance() throws Exception { + updateClusterSettings( + new ClusterSetting(PERSISTENT, Settings.Key.FIELD_TYPE_TOLERANCE.getKeyValue(), "false")); + try { + JSONObject response = new JSONObject(query("SELECT accounts FROM %s")); + verifyDataRows(response, rows(new JSONObject("{\"id\":1}"))); + } finally { + updateClusterSettings( + new ClusterSetting(PERSISTENT, Settings.Key.FIELD_TYPE_TOLERANCE.getKeyValue(), "true")); + } } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index 66f85b0754..d484f3c4d0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -153,6 +153,7 @@ private Settings defaultSettings() { new ImmutableMap.Builder() .put(Key.QUERY_SIZE_LIMIT, 200) .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true) + .put(Key.FIELD_TYPE_TOLERANCE, true) .build(); @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index 96bbae94e5..9e7a5be2b2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -423,7 +423,7 @@ public void test_nested_in_where_as_predicate_expression_with_multiple_condition + " nested(message.dayOfWeek) >= 4"; JSONObject result = executeJdbcRequest(query); assertEquals(2, result.getInt("total")); - verifyDataRows(result, rows("c", "ab", 4), rows("zz", "aa", 6)); + verifyDataRows(result, rows("c", "ab", 4), rows("zz", new JSONArray(List.of("aa", "bb")), 6)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java index 698e185abb..f6951f4a2c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java @@ -167,6 +167,7 @@ private Settings defaultSettings() { .put(Key.QUERY_SIZE_LIMIT, 200) .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1)) .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true) + .put(Key.FIELD_TYPE_TOLERANCE, true) .build(); @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index e736663f60..2c82d3bef3 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -75,6 +75,9 @@ public class OpenSearchExprValueFactory { /** The Mapping of Field and ExprType. */ private final Map typeMapping; + /** Whether to support nested value types (such as arrays) */ + private final boolean fieldTypeTolerance; + /** * Extend existing mapping by new data without overwrite. Called from aggregation only {@see * AggregationQueryBuilder#buildTypeMapping}. @@ -143,8 +146,10 @@ public void extendTypeMapping(Map typeMapping) { .build(); /** Constructor of OpenSearchExprValueFactory. */ - public OpenSearchExprValueFactory(Map typeMapping) { + public OpenSearchExprValueFactory( + Map typeMapping, boolean fieldTypeTolerance) { this.typeMapping = OpenSearchDataType.traverseAndFlatten(typeMapping); + this.fieldTypeTolerance = fieldTypeTolerance; } /** @@ -160,7 +165,7 @@ public ExprValue construct(String jsonString, boolean supportArrays) { new OpenSearchJsonContent(OBJECT_MAPPER.readTree(jsonString)), TOP_PATH, Optional.of(STRUCT), - supportArrays); + fieldTypeTolerance || supportArrays); } catch (JsonProcessingException e) { throw new IllegalStateException(String.format("invalid json: %s.", jsonString), e); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index fff252f3b4..cc2b2d781c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -150,7 +150,9 @@ public OpenSearchQueryRequest(StreamInput in, OpenSearchStorageEngine engine) th } OpenSearchIndex index = (OpenSearchIndex) engine.getTable(null, indexName.toString()); - exprValueFactory = new OpenSearchExprValueFactory(index.getFieldOpenSearchTypes()); + exprValueFactory = + new OpenSearchExprValueFactory( + index.getFieldOpenSearchTypes(), index.isFieldTypeTolerance()); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java index c9490f0767..d793b53fca 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java @@ -178,6 +178,8 @@ public OpenSearchScrollRequest(StreamInput in, OpenSearchStorageEngine engine) includes = in.readStringList(); indexName = new IndexName(in); OpenSearchIndex index = (OpenSearchIndex) engine.getTable(null, indexName.toString()); - exprValueFactory = new OpenSearchExprValueFactory(index.getFieldOpenSearchTypes()); + exprValueFactory = + new OpenSearchExprValueFactory( + index.getFieldOpenSearchTypes(), index.isFieldTypeTolerance()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 1083dbd836..612771eea4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -229,6 +229,13 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting FIELD_TYPE_TOLERANCE_SETTING = + Setting.boolSetting( + Key.FIELD_TYPE_TOLERANCE.getKeyValue(), + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + /** Construct OpenSearchSetting. The OpenSearchSetting must be singleton. */ @SuppressWarnings("unchecked") public OpenSearchSettings(ClusterSettings clusterSettings) { @@ -372,13 +379,19 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { clusterSettings, Key.SESSION_INACTIVITY_TIMEOUT_MILLIS, SESSION_INACTIVITY_TIMEOUT_MILLIS_SETTING, - new Updater((Key.SESSION_INACTIVITY_TIMEOUT_MILLIS))); + new Updater(Key.SESSION_INACTIVITY_TIMEOUT_MILLIS)); register( settingBuilder, clusterSettings, Key.STREAMING_JOB_HOUSEKEEPER_INTERVAL, STREAMING_JOB_HOUSEKEEPER_INTERVAL_SETTING, - new Updater((Key.STREAMING_JOB_HOUSEKEEPER_INTERVAL))); + new Updater(Key.STREAMING_JOB_HOUSEKEEPER_INTERVAL)); + register( + settingBuilder, + clusterSettings, + Key.FIELD_TYPE_TOLERANCE, + FIELD_TYPE_TOLERANCE_SETTING, + new Updater(Key.FIELD_TYPE_TOLERANCE)); defaultSettings = settingBuilder.build(); } @@ -455,6 +468,7 @@ public static List> pluginSettings() { .add(DATASOURCES_LIMIT_SETTING) .add(SESSION_INACTIVITY_TIMEOUT_MILLIS_SETTING) .add(STREAMING_JOB_HOUSEKEEPER_INTERVAL_SETTING) + .add(FIELD_TYPE_TOLERANCE_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index a6fe83c8c4..b8822cd1e8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -177,7 +177,12 @@ private OpenSearchExprValueFactory createExprValueFactory() { Map allFields = new HashMap<>(); getReservedFieldTypes().forEach((k, v) -> allFields.put(k, OpenSearchDataType.of(v))); allFields.putAll(getFieldOpenSearchTypes()); - return new OpenSearchExprValueFactory(allFields); + return new OpenSearchExprValueFactory( + allFields, settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)); + } + + public boolean isFieldTypeTolerance() { + return settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE); } @VisibleForTesting diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java index b17773cb03..74cbd1f167 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java @@ -47,8 +47,6 @@ public class OpenSearchIndexScan extends TableScanOperator implements Serializab /** Search response for current batch. */ private Iterator iterator; - private Settings pluginSettings; - /** Creates index scan based on a provided OpenSearchRequestBuilder. */ public OpenSearchIndexScan( OpenSearchClient client, int maxResponseSize, OpenSearchRequest request) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java index 3a9ff02ba0..460a9b4567 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java @@ -102,7 +102,7 @@ private OpenSearchExprValueFactory buildValueFactory(Set fi Map typeEnv = fields.stream() .collect(toMap(ReferenceExpression::getAttr, e -> OpenSearchDataType.of(e.type()))); - return new OpenSearchExprValueFactory(typeEnv); + return new OpenSearchExprValueFactory(typeEnv, false); } private Environment buildValueEnv( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index ba0fb85422..12a906b25e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -297,7 +297,6 @@ void search() { new SearchHits( new SearchHit[] {searchHit}, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0F)); when(searchHit.getSourceAsString()).thenReturn("{\"id\", 1}"); - when(searchHit.getInnerHits()).thenReturn(null); when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue); // Mock second scroll request followed diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index 82d9e74422..eb2355a36b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -286,7 +286,6 @@ void search() throws IOException { new SearchHits( new SearchHit[] {searchHit}, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0F)); when(searchHit.getSourceAsString()).thenReturn("{\"id\", 1}"); - when(searchHit.getInnerHits()).thenReturn(null); when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue); // Mock second scroll request followed diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index d1bc3c500b..59484f1f17 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -121,7 +121,10 @@ class OpenSearchExprValueFactoryTest { .build(); private final OpenSearchExprValueFactory exprValueFactory = - new OpenSearchExprValueFactory(MAPPING); + new OpenSearchExprValueFactory(MAPPING, true); + + private final OpenSearchExprValueFactory exprValueFactoryNoArrays = + new OpenSearchExprValueFactory(MAPPING, false); @Test public void constructNullValue() { @@ -478,28 +481,40 @@ public void constructArrayOfStrings() { constructFromObject("arrayV", List.of("zz", "au"))); } + @Test + public void constructArrayOfStringsWithArrays() { + assertEquals( + new ExprCollectionValue(List.of(stringValue("zz"), stringValue("au"))), + constructFromObjectWithArraySupport("arrayV", List.of("zz", "au"))); + } + @Test public void constructNestedArraysOfStrings() { assertEquals( new ExprCollectionValue( List.of(collectionValue(List.of("zz", "au")), collectionValue(List.of("ss")))), - tupleValueWithArraySupport("{\"stringV\":[" + "[\"zz\", \"au\"]," + "[\"ss\"]" + "]}") - .get("stringV")); + tupleValueWithArraySupport("{\"stringV\":[ [\"zz\", \"au\"], [\"ss\"] ]}").get("stringV")); } @Test - public void constructNestedArraysOfStringsReturnsFirstIndex() { + public void constructNestedArraysOfStringsReturnsAll() { assertEquals( - stringValue("zz"), - tupleValue("{\"stringV\":[" + "[\"zz\", \"au\"]," + "[\"ss\"]" + "]}").get("stringV")); + new ExprCollectionValue( + List.of( + new ExprCollectionValue(List.of(stringValue("zz"), stringValue("au"))), + new ExprCollectionValue(List.of(stringValue("ss"))))), + tupleValue("{\"stringV\":[[\"zz\", \"au\"],[\"ss\"]]}").get("stringV")); } @Test - public void constructMultiNestedArraysOfStringsReturnsFirstIndex() { + public void constructMultiNestedArraysOfStringsReturnsAll() { assertEquals( - stringValue("z"), - tupleValue("{\"stringV\":" + "[\"z\"," + "[\"s\"]," + "[\"zz\", \"au\"]" + "]}") - .get("stringV")); + new ExprCollectionValue( + List.of( + stringValue("z"), + new ExprCollectionValue(List.of(stringValue("s"))), + new ExprCollectionValue(List.of(stringValue("zz"), stringValue("au"))))), + tupleValue("{\"stringV\":[\"z\",[\"s\"],[\"zz\", \"au\"]]}").get("stringV")); } @Test @@ -594,6 +609,20 @@ public void constructNestedArrayNode() { tupleValueWithArraySupport("{\"nestedV\":[1969,2011]}").get("nestedV")); } + @Test + public void constructNestedArrayNodeNotSupported() { + assertEquals( + Map.of("stringV", stringValue("foo")), + tupleValueWithoutArraySupport("[{\"stringV\":\"foo\"}]")); + } + + @Test + public void constructNestedArrayNodeNotSupportedNoFieldTolerance() { + assertEquals( + Map.of("stringV", stringValue("foo")), + tupleValueWithoutArraySupportNoFieldTolerance("{\"stringV\":\"foo\"}")); + } + @Test public void constructNestedObjectNode() { assertEquals( @@ -617,9 +646,24 @@ public void constructArrayOfGeoPoints() { } @Test - public void constructArrayOfGeoPointsReturnsFirstIndex() { + public void constructArrayOfGeoPointsNoArrays() { assertEquals( new OpenSearchExprGeoPointValue(42.60355556, -97.25263889), + tupleValueWithoutArraySupport( + "{\"geoV\":[" + + "{\"lat\":42.60355556,\"lon\":-97.25263889}," + + "{\"lat\":-33.6123556,\"lon\":66.287449}" + + "]}") + .get("geoV")); + } + + @Test + public void constructArrayOfGeoPointsReturnsAll() { + assertEquals( + new ExprCollectionValue( + List.of( + new OpenSearchExprGeoPointValue(42.60355556, -97.25263889), + new OpenSearchExprGeoPointValue(-33.6123556, 66.287449))), tupleValue( "{\"geoV\":[" + "{\"lat\":42.60355556,\"lon\":-97.25263889}," @@ -629,46 +673,60 @@ public void constructArrayOfGeoPointsReturnsFirstIndex() { } @Test - public void constructArrayOfIPsReturnsFirstIndex() { + public void constructArrayOfIPsReturnsAll() { assertEquals( - new OpenSearchExprIpValue("192.168.0.1"), + new ExprCollectionValue( + List.of( + new OpenSearchExprIpValue("192.168.0.1"), + new OpenSearchExprIpValue("192.168.0.2"))), tupleValue("{\"ipV\":[\"192.168.0.1\",\"192.168.0.2\"]}").get("ipV")); } @Test - public void constructBinaryArrayReturnsFirstIndex() { + public void constructBinaryArrayReturnsAll() { assertEquals( - new OpenSearchExprBinaryValue("U29tZSBiaWsdfsdfgYmxvYg=="), + new ExprCollectionValue( + List.of( + new OpenSearchExprBinaryValue("U29tZSBiaWsdfsdfgYmxvYg=="), + new OpenSearchExprBinaryValue("U987yuhjjiy8jhk9vY+98jjdf"))), tupleValue("{\"binaryV\":[\"U29tZSBiaWsdfsdfgYmxvYg==\",\"U987yuhjjiy8jhk9vY+98jjdf\"]}") .get("binaryV")); } @Test - public void constructArrayOfCustomEpochMillisReturnsFirstIndex() { + public void constructArrayOfCustomEpochMillisReturnsAll() { assertEquals( - new ExprDatetimeValue("2015-01-01 12:10:30"), + new ExprCollectionValue( + List.of( + new ExprTimestampValue("2015-01-01 12:10:30"), + new ExprTimestampValue("1999-11-09 01:09:44"))), tupleValue("{\"customAndEpochMillisV\":[\"2015-01-01 12:10:30\",\"1999-11-09 01:09:44\"]}") .get("customAndEpochMillisV")); } @Test - public void constructArrayOfDateStringsReturnsFirstIndex() { + public void constructArrayOfDateStringsReturnsAll() { assertEquals( - new ExprDateValue("1984-04-12"), + new ExprCollectionValue( + List.of(new ExprDateValue("1984-04-12"), new ExprDateValue("2033-05-03"))), tupleValue("{\"dateStringV\":[\"1984-04-12\",\"2033-05-03\"]}").get("dateStringV")); } @Test - public void constructArrayOfTimeStringsReturnsFirstIndex() { + public void constructArrayOfTimeStringsReturnsAll() { assertEquals( - new ExprTimeValue("12:10:30"), + new ExprCollectionValue( + List.of(new ExprTimeValue("12:10:30"), new ExprTimeValue("18:33:55"))), tupleValue("{\"timeStringV\":[\"12:10:30.000Z\",\"18:33:55.000Z\"]}").get("timeStringV")); } @Test public void constructArrayOfEpochMillis() { assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + new ExprCollectionValue( + List.of( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + new ExprTimestampValue(Instant.ofEpochMilli(1454251113333L)))), tupleValue("{\"dateOrEpochMillisV\":[\"1420070400001\",\"1454251113333\"]}") .get("dateOrEpochMillisV")); } @@ -780,12 +838,75 @@ public void constructBinary() { } /** - * Return the first element if is OpenSearch Array. + * Return the all elements if is OpenSearch Array. * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. */ @Test - public void constructFromOpenSearchArrayReturnFirstElement() { - assertEquals(integerValue(1), tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); + public void constructFromOpenSearchArrayReturnAll() { + assertEquals( + new ExprCollectionValue(List.of(integerValue(1), integerValue(2), integerValue(3))), + tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); + assertEquals( + new ExprCollectionValue( + List.of( + new ExprTupleValue( + new LinkedHashMap() { + { + put("id", integerValue(1)); + put("state", stringValue("WA")); + } + }), + new ExprTupleValue( + new LinkedHashMap() { + { + put("id", integerValue(2)); + put("state", stringValue("CA")); + } + }))), + tupleValue("{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") + .get("structV")); + } + + /** + * Return the all elements if is OpenSearch Array. + * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. + */ + @Test + public void constructFromOpenSearchArrayReturnAllWithArraySupport() { + assertEquals( + new ExprCollectionValue(List.of(integerValue(1), integerValue(2), integerValue(3))), + tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); + assertEquals( + new ExprCollectionValue( + List.of( + new ExprTupleValue( + new LinkedHashMap() { + { + put("id", integerValue(1)); + put("state", stringValue("WA")); + } + }), + new ExprTupleValue( + new LinkedHashMap() { + { + put("id", integerValue(2)); + put("state", stringValue("CA")); + } + }))), + tupleValueWithArraySupport( + "{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") + .get("structV")); + } + + /** + * Return only the first element if is OpenSearch Array. + * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. + */ + @Test + public void constructFromOpenSearchArrayReturnAllWithoutArraySupport() { + assertEquals( + new ExprCollectionValue(List.of(integerValue(1), integerValue(2), integerValue(3))), + tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); assertEquals( new ExprTupleValue( new LinkedHashMap() { @@ -794,7 +915,39 @@ public void constructFromOpenSearchArrayReturnFirstElement() { put("state", stringValue("WA")); } }), - tupleValue("{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") + tupleValueWithoutArraySupport( + "{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") + .get("structV")); + } + + /** + * Return only the first element if is OpenSearch Array. + * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. + */ + @Test + public void constructFromOpenSearchArrayReturnAllWithoutArraySupportNoFieldTolerance() { + assertEquals( + new ExprCollectionValue(List.of(integerValue(1), integerValue(2), integerValue(3))), + tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); + assertEquals( + new ExprCollectionValue( + List.of( + new ExprTupleValue( + new LinkedHashMap() { + { + put("id", integerValue(1)); + put("state", stringValue("WA")); + } + }), + new ExprTupleValue( + new LinkedHashMap() { + { + put("id", integerValue(2)); + put("state", stringValue("CA")); + } + }))), + tupleValueWithoutArraySupportNoFieldTolerance( + "{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") .get("structV")); } @@ -815,7 +968,7 @@ public void noTypeFoundForMapping() { @Test public void constructUnsupportedTypeThrowException() { OpenSearchExprValueFactory exprValueFactory = - new OpenSearchExprValueFactory(Map.of("type", new TestType())); + new OpenSearchExprValueFactory(Map.of("type", new TestType()), true); IllegalStateException exception = assertThrows( IllegalStateException.class, () -> exprValueFactory.construct("{\"type\":1}", false)); @@ -832,7 +985,8 @@ public void constructUnsupportedTypeThrowException() { // it is accepted without overwriting existing data. public void factoryMappingsAreExtendableWithoutOverWrite() throws NoSuchFieldException, IllegalAccessException { - var factory = new OpenSearchExprValueFactory(Map.of("value", OpenSearchDataType.of(INTEGER))); + var factory = + new OpenSearchExprValueFactory(Map.of("value", OpenSearchDataType.of(INTEGER)), true); factory.extendTypeMapping( Map.of( "value", OpenSearchDataType.of(DOUBLE), @@ -860,6 +1014,16 @@ public Map tupleValueWithArraySupport(String jsonString) { return construct.tupleValue(); } + public Map tupleValueWithoutArraySupport(String jsonString) { + final ExprValue construct = exprValueFactoryNoArrays.construct(jsonString, false); + return construct.tupleValue(); + } + + public Map tupleValueWithoutArraySupportNoFieldTolerance(String jsonString) { + final ExprValue construct = exprValueFactoryNoArrays.construct(jsonString, true); + return construct.tupleValue(); + } + private ExprValue constructFromObject(String fieldName, Object value) { return exprValueFactory.construct(fieldName, value, false); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java index 89b51207b5..0847e520cc 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java @@ -20,7 +20,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.action.search.SearchRequest; @@ -72,7 +71,7 @@ public class OpenSearchQueryRequestTest { @Mock private OpenSearchStorageEngine engine; @Mock private PointInTimeBuilder pointInTimeBuilder; - @InjectMocks private OpenSearchQueryRequest serializationRequest; + private OpenSearchQueryRequest serializationRequest; private SearchSourceBuilder sourceBuilderForSerializer; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java index bf87840b60..a2430a671d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java @@ -86,6 +86,7 @@ void setup() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) .thenReturn(true); + lenient().when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)).thenReturn(false); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index 6f4605bc2f..217145a052 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -14,10 +14,12 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -163,7 +165,8 @@ void iterator_metafields() { "_sort", new ExprLongValue(123456L), "_score", new ExprFloatValue(3.75F), "_maxscore", new ExprFloatValue(3.75F))); - List includes = List.of("id1", "_index", "_id", "_routing", "_sort", "_score", "_maxscore"); + List includes = + List.of("id1", "_index", "_id", "_routing", "_sort", "_score", "_maxscore"); int i = 0; for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, includes)) { if (i == 0) { @@ -248,20 +251,15 @@ void iterator_metafields_scoreNaN() { @Test void iterator_with_inner_hits() { + Map innerHits = new HashMap<>(); + innerHits.put("a", mock(SearchHits.class)); + when(searchHit1.getInnerHits()).thenReturn(innerHits); when(searchResponse.getHits()) .thenReturn( new SearchHits( new SearchHit[] {searchHit1}, new TotalHits(2L, TotalHits.Relation.EQUAL_TO), 1.0F)); - when(searchHit1.getInnerHits()) - .thenReturn( - Map.of( - "innerHit", - new SearchHits( - new SearchHit[] {searchHit1}, - new TotalHits(2L, TotalHits.Relation.EQUAL_TO), - 1.0F))); when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue1); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index ef6b86c42a..3f8a07f495 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -10,6 +10,7 @@ import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.hasEntry; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; @@ -82,6 +83,7 @@ void setUp() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) .thenReturn(true); + lenient().when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)).thenReturn(true); } @Test @@ -270,4 +272,13 @@ void implementOtherLogicalOperators() { include), index.implement(plan)); } + + @Test + void isFieldTypeTolerance() { + when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)) + .thenReturn(true) + .thenReturn(false); + assertTrue(index.isFieldTypeTolerance()); + assertFalse(index.isFieldTypeTolerance()); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java index e6a17aceaf..6f923cf5c4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java @@ -59,6 +59,7 @@ void setup() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) .thenReturn(true); + lenient().when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)).thenReturn(true); } @Mock private OpenSearchClient client; @@ -67,7 +68,8 @@ void setup() { new OpenSearchExprValueFactory( Map.of( "name", OpenSearchDataType.of(STRING), - "department", OpenSearchDataType.of(STRING))); + "department", OpenSearchDataType.of(STRING)), + true); @Test void query_empty_result() { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java index e680c6b3a6..da30442bae 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java @@ -78,13 +78,15 @@ class OpenSearchIndexScanTest { private final OpenSearchExprValueFactory exprValueFactory = new OpenSearchExprValueFactory( Map.of( - "name", OpenSearchDataType.of(STRING), "department", OpenSearchDataType.of(STRING))); + "name", OpenSearchDataType.of(STRING), "department", OpenSearchDataType.of(STRING)), + true); @BeforeEach void setup() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) .thenReturn(true); + lenient().when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)).thenReturn(true); } @Test From 81e835bc85b4f7c31dd2cb1aaaa955527b5be1cd Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 25 Oct 2024 15:37:23 -0700 Subject: [PATCH 27/32] [Manual Backport] Improve error handling for some more edge cases (#3080) (#3115) * Improve error handling for some more edge cases (#3080) * Add failing tests Signed-off-by: Simeon Widdis * Fix the first test Signed-off-by: Simeon Widdis * Revise the tests Signed-off-by: Simeon Widdis * Fix wildcard tests Signed-off-by: Simeon Widdis * Add license header Signed-off-by: Simeon Widdis * Fix rerunning SQL parsing Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis * Fix test imports Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .../sql/legacy/MalformedQueryIT.java | 82 +++++++++++++++++++ .../legacy/query/OpenSearchActionFactory.java | 10 ++- .../matchtoterm/TermFieldRewriter.java | 17 ++++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java new file mode 100644 index 0000000000..a0aa1b08a9 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy; + +import java.io.IOException; +import java.util.Locale; +import org.apache.http.ParseException; +import org.apache.http.util.EntityUtils; +import org.json.JSONObject; +import org.junit.Assert; +import org.opensearch.client.ResponseException; + +/** Tests for clean handling of various types of invalid queries */ +public class MalformedQueryIT extends SQLIntegTestCase { + @Override + protected void init() throws Exception { + loadIndex(Index.BANK); + loadIndex(Index.BANK_TWO); + } + + public void testJoinWithInvalidCondition() throws IOException, ParseException { + ResponseException result = + assertThrows( + "Expected Join query with malformed 'ON' to raise error, but didn't", + ResponseException.class, + () -> + executeQuery( + String.format( + Locale.ROOT, + "SELECT a.firstname, b.age FROM %s AS a INNER JOIN %s AS b %%" + + " a.account_number=b.account_number", + TestsConstants.TEST_INDEX_BANK, + TestsConstants.TEST_INDEX_BANK_TWO))); + var errMsg = new JSONObject(EntityUtils.toString(result.getResponse().getEntity())); + + Assert.assertEquals("SqlParseException", errMsg.getJSONObject("error").getString("type")); + Assert.assertEquals(400, errMsg.getInt("status")); + } + + public void testWrappedWildcardInSubquery() throws IOException, ParseException { + ResponseException result = + assertThrows( + "Expected wildcard subquery to raise error, but didn't", + ResponseException.class, + () -> + executeQuery( + String.format( + Locale.ROOT, + "SELECT a.first_name FROM %s AS a WHERE a.age IN (SELECT age FROM" + + " `opensearch-sql_test_index_*` WHERE age > 30)", + TestsConstants.TEST_INDEX_BANK, + TestsConstants.TEST_INDEX_BANK_TWO))); + var errMsg = new JSONObject(EntityUtils.toString(result.getResponse().getEntity())); + System.err.println("Full response: " + errMsg); + + Assert.assertEquals("IndexNotFoundException", errMsg.getJSONObject("error").getString("type")); + Assert.assertEquals(404, errMsg.getInt("status")); + } + + public void testUnwrappedWildcardInSubquery() throws IOException, ParseException { + ResponseException result = + assertThrows( + "Expected wildcard subquery to raise error, but didn't", + ResponseException.class, + () -> + executeQuery( + String.format( + Locale.ROOT, + "SELECT a.first_name FROM %s AS a WHERE a.age IN (SELECT age FROM * WHERE" + + " age > 30)", + TestsConstants.TEST_INDEX_BANK, + TestsConstants.TEST_INDEX_BANK_TWO))); + var errMsg = new JSONObject(EntityUtils.toString(result.getResponse().getEntity())); + System.err.println("Full response: " + errMsg); + + Assert.assertEquals("IndexNotFoundException", errMsg.getJSONObject("error").getString("type")); + Assert.assertEquals(404, errMsg.getInt("status")); + } +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java index b9a7c9f218..a5bfeebfbd 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.domain.IndexStatement.StatementType; import static org.opensearch.sql.legacy.utils.Util.toSqlExpr; +import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr; import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; @@ -86,7 +87,14 @@ public static QueryAction create(Client client, QueryActionRequest request) switch (getFirstWord(sql)) { case "SELECT": - SQLQueryExpr sqlExpr = (SQLQueryExpr) toSqlExpr(sql); + SQLExpr rawExpr = toSqlExpr(sql); + if (!(rawExpr instanceof SQLQueryExpr)) { + throw new SqlParseException( + "Expected a query expression, but found a " + + rawExpr.getClass().getSimpleName() + + ". The query is not runnable."); + } + SQLQueryExpr sqlExpr = (SQLQueryExpr) rawExpr; RewriteRuleExecutor ruleExecutor = RewriteRuleExecutor.builder() diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java index 1200c8befb..344d1a268b 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Stream; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.esdomain.mapping.FieldMappings; import org.opensearch.sql.legacy.esdomain.mapping.IndexMappings; @@ -122,7 +123,23 @@ public boolean visit(SQLIdentifierExpr expr) { String fullFieldName = arr[1]; String index = curScope().getAliases().get(alias); + if (index == null) { + throw new IndexNotFoundException( + String.format( + "The requested table '%s' does not correspond to any known index. Only indices or" + + " table aliases are allowed.", + alias.replaceFirst("_\\d+$", ""))); + } + FieldMappings fieldMappings = curScope().getMapper().mapping(index); + if (fieldMappings == null) { + throw new IndexNotFoundException( + String.format( + "The index '%s' could not be found. Note that wildcard indices are not permitted" + + " in SQL.", + index)); + } + if (fieldMappings.has(fullFieldName)) { source = fieldMappings.mapping(fullFieldName); } else { From 8cf82e9f23668437bca36c4215728cf61016e33b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 09:17:24 -0700 Subject: [PATCH 28/32] Add release notes for v2.18.0 (#3125) (#3126) * Add release notes for 2.18 * Remove all backport labels --------- (cherry picked from commit f00244eccafc50226950afa46dfe44defdfd9142) Signed-off-by: Chen Dai Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../opensearch-sql.release-notes-2.18.0.0.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-2.18.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-2.18.0.0.md b/release-notes/opensearch-sql.release-notes-2.18.0.0.md new file mode 100644 index 0000000000..1acd0c7d21 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-2.18.0.0.md @@ -0,0 +1,20 @@ +Compatible with OpenSearch and OpenSearch Dashboards Version 2.18.0 + +### Features + +* Backport #2981 to 2.x ([#3111](https://github.com/opensearch-project/sql/pull/3111)) + +### Bug Fixes + +* Improve error handling for some more edge cases ([#3112](https://github.com/opensearch-project/sql/pull/3112)) +* Resolve Alias Issues in Legacy SQL with Filters ([#3109](https://github.com/opensearch-project/sql/pull/3109)) +* Bug Fixes for minor issues with SQL PIT refactor ([#3108](https://github.com/opensearch-project/sql/pull/3108)) +* Correct regular expression range ([#3107](https://github.com/opensearch-project/sql/pull/3107)) +* SQL pagination should work with the `pretty` parameter ([#3106](https://github.com/opensearch-project/sql/pull/3106)) +* Improve error handling for malformed query cursors ([#3084](https://github.com/opensearch-project/sql/pull/3084)) +* Remove scheduler index from SystemIndexDescriptor ([#3097](https://github.com/opensearch-project/sql/pull/3097)) + +### Maintenance + +* bump commons-io to 2.14.0 ([#3091](https://github.com/opensearch-project/sql/pull/3091)) +* Fix tests on 2.18 ([#3113](https://github.com/opensearch-project/sql/pull/3113)) From c509759f661fbe4e14107090bf19cd42cf7e2100 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:05:07 -0700 Subject: [PATCH 29/32] Add validation method for Flint extension queries and wire it into the dispatcher (#3096) (#3130) * Add validation method for Flint extension queries and wire it into the dispatcher * Add unit test * Run spotless * Fix NPE in test * Add java doc --------- (cherry picked from commit 3b8661201efe893cd80da5f8dbd3b21333a8eda9) Signed-off-by: Chase Engelbrecht Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../sql/spark/dispatcher/SparkQueryDispatcher.java | 1 + .../sql/spark/validator/SQLQueryValidator.java | 8 ++++++++ .../sql/spark/dispatcher/SparkQueryDispatcherTest.java | 5 +++++ .../sql/spark/validator/SQLQueryValidatorTest.java | 10 ++++++++++ 4 files changed, 24 insertions(+) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index ff8c8d1fe8..5f88ea9ca0 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -51,6 +51,7 @@ public DispatchQueryResponse dispatch( String query = dispatchQueryRequest.getQuery(); if (SQLQueryUtils.isFlintExtensionQuery(query)) { + sqlQueryValidator.validateFlintExtensionQuery(query, dataSourceMetadata.getConnector()); return handleFlintExtensionQuery( dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java index f387cbad25..5d934411bf 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java @@ -36,4 +36,12 @@ public void validate(String sqlQuery, DataSourceType datasourceType) { throw e; } } + + /** + * Validates a query from the Flint extension grammar. The method is currently a no-op. + * + * @param sqlQuery The Flint extension query to be validated + * @param dataSourceType The type of the datasource the query is being run on + */ + public void validateFlintExtensionQuery(String sqlQuery, DataSourceType dataSourceType) {} } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 343dc3040a..405fdf511d 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -641,6 +641,11 @@ void testDispatchVacuumIndexQuery() { @Test void testDispatchRecoverIndexQuery() { + DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) + .thenReturn(dataSourceMetadata); + String query = "RECOVER INDEX JOB `flint_spark_catalog_default_test_skipping_index`"; Assertions.assertThrows( IllegalArgumentException.class, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java index 695a083809..fbc00109da 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -5,11 +5,13 @@ package org.opensearch.sql.spark.validator; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import java.util.Arrays; +import java.util.UUID; import lombok.AllArgsConstructor; import lombok.Getter; import org.antlr.v4.runtime.CommonTokenStream; @@ -561,6 +563,14 @@ void testSecurityLakeQueries() { v.ng(TestElement.INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS); } + @Test + void testValidateFlintExtensionQuery() { + assertDoesNotThrow( + () -> + sqlQueryValidator.validateFlintExtensionQuery( + UUID.randomUUID().toString(), DataSourceType.SECURITY_LAKE)); + } + @AllArgsConstructor private static class VerifyValidator { private final SQLQueryValidator validator; From c942259cc8b76a83c935d3d8fae7a237109ea55a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 31 Oct 2024 10:34:22 -0700 Subject: [PATCH 30/32] Call updateState when query is cancelled (#3139) (#3144) * Call updateState when query is cancelled * Fix code style --------- (cherry picked from commit 5716cab002085342241023ef11b3a72f5f9e0151) Signed-off-by: Tomoyuki Morita Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../AsyncQueryExecutorServiceImpl.java | 7 +++++- .../AsyncQueryJobMetadataStorageService.java | 6 +++++ .../AsyncQueryExecutorServiceImplTest.java | 25 ++++++++++--------- ...chAsyncQueryJobMetadataStorageService.java | 9 +++++++ 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java index 0e9e128896..0639768354 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java @@ -19,6 +19,7 @@ import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.config.SparkExecutionEngineConfig; import org.opensearch.sql.spark.config.SparkExecutionEngineConfigSupplier; import org.opensearch.sql.spark.dispatcher.SparkQueryDispatcher; @@ -116,7 +117,11 @@ public String cancelQuery(String queryId, AsyncQueryRequestContext asyncQueryReq Optional asyncQueryJobMetadata = asyncQueryJobMetadataStorageService.getJobMetadata(queryId); if (asyncQueryJobMetadata.isPresent()) { - return sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata.get(), asyncQueryRequestContext); + String result = + sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata.get(), asyncQueryRequestContext); + asyncQueryJobMetadataStorageService.updateState( + asyncQueryJobMetadata.get(), QueryState.CANCELLED, asyncQueryRequestContext); + return result; } throw new AsyncQueryNotFoundException(String.format("QueryId: %s not found", queryId)); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java index b4e94c984d..86e925f58f 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java @@ -10,6 +10,7 @@ import java.util.Optional; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; public interface AsyncQueryJobMetadataStorageService { @@ -17,5 +18,10 @@ void storeJobMetadata( AsyncQueryJobMetadata asyncQueryJobMetadata, AsyncQueryRequestContext asyncQueryRequestContext); + void updateState( + AsyncQueryJobMetadata asyncQueryJobMetadata, + QueryState newState, + AsyncQueryRequestContext asyncQueryRequestContext); + Optional getJobMetadata(String jobId); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java index 73850db83d..3177c335d9 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.spark.asyncquery; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; @@ -33,6 +34,7 @@ import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.config.SparkExecutionEngineConfig; import org.opensearch.sql.spark.config.SparkExecutionEngineConfigSupplier; import org.opensearch.sql.spark.config.SparkSubmitParameterModifier; @@ -109,7 +111,7 @@ void testCreateAsyncQuery() { .getSparkExecutionEngineConfig(asyncQueryRequestContext); verify(sparkQueryDispatcher, times(1)) .dispatch(expectedDispatchQueryRequest, asyncQueryRequestContext); - Assertions.assertEquals(QUERY_ID, createAsyncQueryResponse.getQueryId()); + assertEquals(QUERY_ID, createAsyncQueryResponse.getQueryId()); } @Test @@ -153,8 +155,7 @@ void testGetAsyncQueryResultsWithJobNotFoundException() { AsyncQueryNotFoundException.class, () -> jobExecutorService.getAsyncQueryResults(EMR_JOB_ID, asyncQueryRequestContext)); - Assertions.assertEquals( - "QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); + assertEquals("QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); verifyNoInteractions(sparkQueryDispatcher); verifyNoInteractions(sparkExecutionEngineConfigSupplier); } @@ -174,7 +175,7 @@ void testGetAsyncQueryResultsWithInProgressJob() { Assertions.assertNull(asyncQueryExecutionResponse.getResults()); Assertions.assertNull(asyncQueryExecutionResponse.getSchema()); - Assertions.assertEquals("PENDING", asyncQueryExecutionResponse.getStatus()); + assertEquals("PENDING", asyncQueryExecutionResponse.getStatus()); verifyNoInteractions(sparkExecutionEngineConfigSupplier); } @@ -191,11 +192,10 @@ void testGetAsyncQueryResultsWithSuccessJob() throws IOException { AsyncQueryExecutionResponse asyncQueryExecutionResponse = jobExecutorService.getAsyncQueryResults(EMR_JOB_ID, asyncQueryRequestContext); - Assertions.assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); - Assertions.assertEquals(1, asyncQueryExecutionResponse.getSchema().getColumns().size()); - Assertions.assertEquals( - "1", asyncQueryExecutionResponse.getSchema().getColumns().get(0).getName()); - Assertions.assertEquals( + assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); + assertEquals(1, asyncQueryExecutionResponse.getSchema().getColumns().size()); + assertEquals("1", asyncQueryExecutionResponse.getSchema().getColumns().get(0).getName()); + assertEquals( 1, ((HashMap) asyncQueryExecutionResponse.getResults().get(0).value()) .get("1")); @@ -212,8 +212,7 @@ void testCancelJobWithJobNotFound() { AsyncQueryNotFoundException.class, () -> jobExecutorService.cancelQuery(EMR_JOB_ID, asyncQueryRequestContext)); - Assertions.assertEquals( - "QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); + assertEquals("QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); verifyNoInteractions(sparkQueryDispatcher); verifyNoInteractions(sparkExecutionEngineConfigSupplier); } @@ -227,7 +226,9 @@ void testCancelJob() { String jobId = jobExecutorService.cancelQuery(EMR_JOB_ID, asyncQueryRequestContext); - Assertions.assertEquals(EMR_JOB_ID, jobId); + assertEquals(EMR_JOB_ID, jobId); + verify(asyncQueryJobMetadataStorageService) + .updateState(any(), eq(QueryState.CANCELLED), eq(asyncQueryRequestContext)); verifyNoInteractions(sparkExecutionEngineConfigSupplier); } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java b/async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java index 4847c8e00f..eb377a5cff 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java @@ -12,6 +12,7 @@ import org.opensearch.sql.spark.asyncquery.exceptions.AsyncQueryNotFoundException; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.execution.statestore.OpenSearchStateStoreUtil; import org.opensearch.sql.spark.execution.statestore.StateStore; import org.opensearch.sql.spark.execution.xcontent.AsyncQueryJobMetadataXContentSerializer; @@ -39,6 +40,14 @@ public void storeJobMetadata( OpenSearchStateStoreUtil.getIndexName(asyncQueryJobMetadata.getDatasourceName())); } + @Override + public void updateState( + AsyncQueryJobMetadata asyncQueryJobMetadata, + QueryState newState, + AsyncQueryRequestContext asyncQueryRequestContext) { + // NoOp since AsyncQueryJobMetadata record does not store state now + } + private String mapIdToDocumentId(String id) { return "qid" + id; } From 30a2072711fdf384bf83567724b61e8399be0348 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:02:49 -0700 Subject: [PATCH 31/32] Added documentation for the plugins.query.field_type_tolerance setting (#1300) (#3118) (#3143) (cherry picked from commit 9bb4c08acdc5e0412bc5d35c1dc6ee6d219d7245) Signed-off-by: Norman Jordan Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- docs/user/admin/settings.rst | 58 +++++++++++++++++++++++++ docs/user/limitations/limitations.rst | 29 +++++++++++++ doctest/test_data/multi_value_long.json | 5 +++ 3 files changed, 92 insertions(+) create mode 100644 doctest/test_data/multi_value_long.json diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 71718d1726..cbcb4f329d 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -824,3 +824,61 @@ To Re-enable Data Sources::: } } +plugins.query.field_type_tolerance +================================== + +Description +----------- + +This setting controls whether preserve arrays. If this setting is set to false, then an array is reduced +to the first non array value of any level of nesting. + +1. The default value is true (preserve arrays) +2. This setting is node scope +3. This setting can be updated dynamically + +Querying a field containing array values will return the full array values:: + + os> SELECT accounts FROM people; + fetched rows / total rows = 1/1 + +-----------------------+ + | accounts | + +-----------------------+ + | [{'id': 1},{'id': 2}] | + +-----------------------+ + +Disable field type tolerance:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.query.field_type_tolerance" : false + } + }' + +When field type tolerance is disabled, arrays are collapsed to the first non array value:: + + os> SELECT accounts FROM people; + fetched rows / total rows = 1/1 + +-----------+ + | accounts | + +-----------+ + | {'id': 1} | + +-----------+ + +Reenable field type tolerance:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.query.field_type_tolerance" : true + } + }' + +Limitations: +------------ +OpenSearch does not natively support the ARRAY data type but does allow multi-value fields implicitly. The +SQL/PPL plugin adheres strictly to the data type semantics defined in index mappings. When parsing OpenSearch +responses, it expects data to match the declared type and does not account for data in array format. If the +plugins.query.field_type_tolerance setting is enabled, the SQL/PPL plugin will handle array datasets by returning +scalar data types, allowing basic queries (e.g., SELECT * FROM tbl WHERE condition). However, using multi-value +fields in expressions or functions will result in exceptions. If this setting is disabled or absent, only the +first element of an array is returned, preserving the default behavior. \ No newline at end of file diff --git a/docs/user/limitations/limitations.rst b/docs/user/limitations/limitations.rst index 8ce75a0e25..22ad3c2a17 100644 --- a/docs/user/limitations/limitations.rst +++ b/docs/user/limitations/limitations.rst @@ -101,3 +101,32 @@ The response in JDBC format with cursor id:: } The query with `aggregation` and `join` does not support pagination for now. + +Limitations on Using Multi-valued Fields +======================================== + +OpenSearch does not natively support the ARRAY data type but does allow multi-value fields implicitly. The +SQL/PPL plugin adheres strictly to the data type semantics defined in index mappings. When parsing OpenSearch +responses, it expects data to match the declared type and does not account for data in array format. If the +plugins.query.field_type_tolerance setting is enabled, the SQL/PPL plugin will handle array datasets by returning +scalar data types, allowing basic queries (e.g., SELECT * FROM tbl WHERE condition). However, using multi-value +fields in expressions or functions will result in exceptions. If this setting is disabled or absent, only the +first element of an array is returned, preserving the default behavior. + +For example, the following query tries to calculate the absolute value of a field that contains arrays of +longs:: + + POST _plugins/_sql/ + { + "query": "SELECT id, ABS(long_array) FROM multi_value_long" + } +The response in JSON format is:: + + { + "error": { + "reason": "Invalid SQL query", + "details": "invalid to get longValue from value of type ARRAY", + "type": "ExpressionEvaluationException" + }, + "status": 400 + } diff --git a/doctest/test_data/multi_value_long.json b/doctest/test_data/multi_value_long.json new file mode 100644 index 0000000000..3c139630f6 --- /dev/null +++ b/doctest/test_data/multi_value_long.json @@ -0,0 +1,5 @@ +{"id": 1, "long_array": [1, 2]} +{"id": 2, "long_array": [3, 4]} +{"id": 3, "long_array": [1, 5]} +{"id": 4, "long_array": [1, 2]} +{"id": 5, "long_array": [2, 3]} \ No newline at end of file From 1be7b30b2a820f59ce690937ca4def596345b846 Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Thu, 7 Nov 2024 02:35:37 +0800 Subject: [PATCH 32/32] [Backport 2.x] Fix a regression issue of parsing datetime string with custom time format in Span (#3079) (#3146) * Fix a regression issue of parsing datetime string with custom time format in Span (#3079) (cherry picked from commit 4ff1fe3cb48a17ff17b0cf134131283fc0282205) * fix compile error Signed-off-by: Lantao Jin * fix coverage Signed-off-by: Lantao Jin --------- Signed-off-by: Lantao Jin --- .../planner/physical/collector/Rounding.java | 8 +-- .../physical/collector/RoundingTest.java | 65 +++++++++++++++++++ .../sql/ppl/DateTimeImplementationIT.java | 37 +++++++++++ .../value/OpenSearchExprValueFactory.java | 2 +- 4 files changed, 107 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java index 81a1a0230f..b16e52fafc 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java @@ -49,16 +49,16 @@ public static Rounding createRounding(SpanExpression span) { if (DOUBLE.isCompatible(type)) { return new DoubleRounding(interval); } - if (type.equals(DATETIME)) { + if (type.equals(DATETIME) || type.typeName().equalsIgnoreCase(DATETIME.typeName())) { return new DatetimeRounding(interval, span.getUnit().getName()); } - if (type.equals(TIMESTAMP)) { + if (type.equals(TIMESTAMP) || type.typeName().equalsIgnoreCase(TIMESTAMP.typeName())) { return new TimestampRounding(interval, span.getUnit().getName()); } - if (type.equals(DATE)) { + if (type.equals(DATE) || type.typeName().equalsIgnoreCase(DATE.typeName())) { return new DateRounding(interval, span.getUnit().getName()); } - if (type.equals(TIME)) { + if (type.equals(TIME) || type.typeName().equalsIgnoreCase(TIME.typeName())) { return new TimeRounding(interval, span.getUnit().getName()); } return new UnknownRounding(); diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java index 3a2601a874..467a4e1abf 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java @@ -5,14 +5,18 @@ package org.opensearch.sql.planner.physical.collector; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import org.junit.jupiter.api.Test; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.span.SpanExpression; @@ -26,6 +30,39 @@ void time_rounding_illegal_span() { ExpressionEvaluationException.class, () -> rounding.round(new ExprTimeValue("23:30:00"))); } + @Test + void datetime_rounding_span() { + SpanExpression dateSpan = DSL.span(DSL.ref("date", DATE), DSL.literal(1), "d"); + Rounding rounding = Rounding.createRounding(dateSpan); + assertInstanceOf(Rounding.DateRounding.class, rounding); + SpanExpression timeSpan = DSL.span(DSL.ref("time", TIME), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timeSpan); + assertInstanceOf(Rounding.TimeRounding.class, rounding); + SpanExpression timestampSpan = DSL.span(DSL.ref("timestamp", TIMESTAMP), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timestampSpan); + assertInstanceOf(Rounding.TimestampRounding.class, rounding); + } + + @Test + void datetime_rounding_non_core_type_span() { + SpanExpression dateSpan = + DSL.span(DSL.ref("date", new MockDateExprType()), DSL.literal(1), "d"); + Rounding rounding = Rounding.createRounding(dateSpan); + assertInstanceOf(Rounding.DateRounding.class, rounding); + SpanExpression timeSpan = + DSL.span(DSL.ref("time", new MockTimeExprType()), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timeSpan); + assertInstanceOf(Rounding.TimeRounding.class, rounding); + SpanExpression timestampSpan = + DSL.span(DSL.ref("timestamp", new MockTimestampExprType()), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timestampSpan); + assertInstanceOf(Rounding.TimestampRounding.class, rounding); + SpanExpression datetimeSpan = + DSL.span(DSL.ref("datetime", new MockDateTimeExprType()), DSL.literal(1), "h"); + rounding = Rounding.createRounding(datetimeSpan); + assertInstanceOf(Rounding.DatetimeRounding.class, rounding); + } + @Test void round_unknown_type() { SpanExpression span = DSL.span(DSL.ref("unknown", STRING), DSL.literal(1), ""); @@ -41,4 +78,32 @@ void resolve() { () -> Rounding.DateTimeUnit.resolve(illegalUnit), "Unable to resolve unit " + illegalUnit); } + + static class MockDateExprType implements ExprType { + @Override + public String typeName() { + return "DATE"; + } + } + + static class MockTimeExprType implements ExprType { + @Override + public String typeName() { + return "TIME"; + } + } + + static class MockTimestampExprType implements ExprType { + @Override + public String typeName() { + return "TIMESTAMP"; + } + } + + static class MockDateTimeExprType implements ExprType { + @Override + public String typeName() { + return "DATETIME"; + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java index dd86470a39..9c7c31e99d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java @@ -6,8 +6,10 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySome; @@ -20,6 +22,7 @@ public class DateTimeImplementationIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.DATE); + loadIndex(Index.DATE_FORMATS); } @Test @@ -176,4 +179,38 @@ public void nullDateTimeInvalidDateValueMonth() throws IOException { verifySchema(result, schema("f", null, "datetime")); verifySome(result.getJSONArray("datarows"), rows(new Object[] {null})); } + + @Test + public void testSpanDatetimeWithCustomFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval a = 1 | stats count() as cnt by span(yyyy-MM-dd, 1d) as span", + TEST_INDEX_DATE_FORMATS)); + verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "date")); + verifyDataRows(result, rows(2, "1984-04-12")); + } + + @Test + public void testSpanDatetimeWithEpochMillisFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval a = 1 | stats count() as cnt by span(epoch_millis, 1d) as span", + TEST_INDEX_DATE_FORMATS)); + verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "timestamp")); + verifyDataRows(result, rows(2, "1984-04-12 00:00:00")); + } + + @Test + public void testSpanDatetimeWithDisjunctiveDifferentFormats() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval a = 1 | stats count() as cnt by span(yyyy-MM-dd_OR_epoch_millis," + + " 1d) as span", + TEST_INDEX_DATE_FORMATS)); + verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "timestamp")); + verifyDataRows(result, rows(2, "1984-04-12 00:00:00")); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 2c82d3bef3..49ac4bd870 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -295,7 +295,7 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) } } else { // custom format - return parseDateTimeString(value.stringValue(), dt); + return parseDateTimeString(value.objectValue().toString(), dt); } } if (value.isString()) {