From d5634216f3d6cbddf1fe296ef14dad634c495e1f Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Thu, 23 May 2024 16:50:57 +0200 Subject: [PATCH 1/3] Run tests with more parallelism During migration from TestNG to JUnit `@TestInstance(PER_CLASS)` annotation was added, but it implies single-threaded execution. Restore previous parallelism: either add `@Execution(CONCURRENT)` or inherit it from base class. --- .../src/test/java/io/trino/cost/TestFilterStatsRule.java | 3 --- .../execution/TestConnectorTracingContextPropagation.java | 3 --- .../optimizations/TestRemoveUnsupportedDynamicFilters.java | 3 --- .../src/test/java/io/trino/sql/query/TestJsonTable.java | 3 +++ .../trino/plugin/clickhouse/BaseClickHouseTypeMapping.java | 3 +++ .../deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java | 3 --- .../plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java | 3 ++- .../deltalake/BaseDeltaLakeRegisterTableProcedureTest.java | 3 +++ .../deltalake/BaseDeltaLakeSharedMetastoreViewsTest.java | 3 +++ ...seDeltaLakeSharedMetastoreWithTableRedirectionsTest.java | 3 --- .../deltalake/TestDeltaLakeAdlsConnectorSmokeTest.java | 3 --- .../io/trino/plugin/deltalake/TestDeltaLakeAdlsStorage.java | 3 --- .../deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java | 3 --- .../deltalake/TestDeltaLakeGcsConnectorSmokeTest.java | 6 ------ .../deltalake/TestDeltaLakeLocalConcurrentWritesTest.java | 3 --- .../trino/plugin/deltalake/TestDeltaLakePartitioning.java | 3 --- .../deltalake/TestDeltaLakeProjectionPushdownPlans.java | 3 --- .../trino/plugin/deltalake/TestDeltaLakeReadTimestamps.java | 3 --- ...stDeltaLakeSharedFileMetastoreWithTableRedirections.java | 3 --- ...stDeltaLakeSharedGlueMetastoreWithTableRedirections.java | 3 --- .../TestDeltaLakeSharedHiveMetastoreWithViews.java | 3 --- .../plugin/deltalake/TestDeltaLakeTableStatistics.java | 3 --- ...tDeltaLakeTableWithCustomLocationUsingGlueMetastore.java | 3 --- .../java/io/trino/plugin/deltalake/TestSplitPruning.java | 3 --- .../TestDeltaLakeConcurrentModificationGlueMetastore.java | 3 --- .../glue/TestDeltaLakeRenameToWithGlueMetastore.java | 3 --- .../metastore/glue/TestDeltaLakeViewsGlueMetastore.java | 3 --- .../src/test/java/io/trino/plugin/hive/TestShowStats.java | 3 --- .../trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java | 3 --- .../io/trino/plugin/iceberg/BaseIcebergConnectorTest.java | 2 -- .../io/trino/plugin/iceberg/BaseIcebergSystemTables.java | 3 --- .../trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java | 3 --- .../plugin/iceberg/TestIcebergGcsConnectorSmokeTest.java | 3 --- .../io/trino/plugin/iceberg/TestIcebergMetadataListing.java | 3 --- .../trino/plugin/iceberg/TestIcebergReadVersionedTable.java | 3 --- .../iceberg/TestIcebergReadVersionedTableByTemporal.java | 3 --- .../io/trino/plugin/iceberg/TestIcebergSplitSource.java | 3 --- .../iceberg/TestIcebergTableWithExternalLocation.java | 3 --- .../test/java/io/trino/plugin/iceberg/TestIcebergV2.java | 3 --- .../glue/TestIcebergGlueCatalogConnectorSmokeTest.java | 3 --- .../catalog/glue/TestIcebergGlueCatalogSkipArchive.java | 3 --- .../thrift/integration/TestThriftProjectionPushdown.java | 3 --- .../test/java/io/trino/tests/TestQueryPlanDeterminism.java | 3 --- 43 files changed, 14 insertions(+), 117 deletions(-) diff --git a/core/trino-main/src/test/java/io/trino/cost/TestFilterStatsRule.java b/core/trino-main/src/test/java/io/trino/cost/TestFilterStatsRule.java index 1b73ef7e69f2..0c2eb6802afe 100644 --- a/core/trino-main/src/test/java/io/trino/cost/TestFilterStatsRule.java +++ b/core/trino-main/src/test/java/io/trino/cost/TestFilterStatsRule.java @@ -22,16 +22,13 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.DoubleType.DOUBLE; import static io.trino.spi.type.IntegerType.INTEGER; import static io.trino.sql.ir.Comparison.Operator.EQUAL; import static io.trino.testing.TestingSession.testSessionBuilder; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestFilterStatsRule extends BaseStatsCalculatorTest { diff --git a/core/trino-main/src/test/java/io/trino/execution/TestConnectorTracingContextPropagation.java b/core/trino-main/src/test/java/io/trino/execution/TestConnectorTracingContextPropagation.java index 5f5c1478a152..87b0b26c7544 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestConnectorTracingContextPropagation.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestConnectorTracingContextPropagation.java @@ -21,16 +21,13 @@ import io.trino.testing.QueryRunner; import io.trino.testing.StandaloneQueryRunner; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import static io.trino.testing.TestingSession.testSessionBuilder; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestConnectorTracingContextPropagation { private static final String CATALOG_NAME = "test_catalog"; diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/optimizations/TestRemoveUnsupportedDynamicFilters.java b/core/trino-main/src/test/java/io/trino/sql/planner/optimizations/TestRemoveUnsupportedDynamicFilters.java index 1e9554824391..2c9df15ba3b6 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/optimizations/TestRemoveUnsupportedDynamicFilters.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/optimizations/TestRemoveUnsupportedDynamicFilters.java @@ -53,7 +53,6 @@ import io.trino.testing.TestingTransactionHandle; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.util.Optional; @@ -76,9 +75,7 @@ import static io.trino.sql.planner.assertions.PlanMatchPattern.values; import static io.trino.sql.planner.plan.JoinType.INNER; import static io.trino.type.UnknownType.UNKNOWN; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestRemoveUnsupportedDynamicFilters extends BasePlanTest { diff --git a/core/trino-main/src/test/java/io/trino/sql/query/TestJsonTable.java b/core/trino-main/src/test/java/io/trino/sql/query/TestJsonTable.java index a1aecaf8ecde..0355c6526783 100644 --- a/core/trino-main/src/test/java/io/trino/sql/query/TestJsonTable.java +++ b/core/trino-main/src/test/java/io/trino/sql/query/TestJsonTable.java @@ -18,14 +18,17 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; import static com.google.common.io.BaseEncoding.base16; import static io.trino.spi.StandardErrorCode.PATH_EVALUATION_ERROR; import static java.nio.charset.StandardCharsets.UTF_16LE; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; @TestInstance(PER_CLASS) +@Execution(CONCURRENT) public class TestJsonTable { private QueryAssertions assertions; diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseTypeMapping.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseTypeMapping.java index 2d33cf15bc59..76c988fa541f 100644 --- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseTypeMapping.java +++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseTypeMapping.java @@ -30,6 +30,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; import java.time.LocalDate; import java.time.LocalDateTime; @@ -58,8 +59,10 @@ import static java.lang.String.format; import static java.time.ZoneOffset.UTC; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; @TestInstance(PER_CLASS) +@Execution(CONCURRENT) public abstract class BaseClickHouseTypeMapping extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java index e7b1529a8f1d..8798886b194c 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java @@ -17,15 +17,12 @@ import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.TestInstance; import java.util.List; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.String.format; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public abstract class BaseDeltaLakeAwsConnectorSmokeTest extends BaseDeltaLakeConnectorSmokeTest { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java index f84855700440..727c41aa4514 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java @@ -45,6 +45,7 @@ import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; import java.util.ArrayList; import java.util.List; @@ -100,8 +101,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.abort; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; -@TestInstance(PER_CLASS) public abstract class BaseDeltaLakeConnectorSmokeTest extends BaseConnectorSmokeTest { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeRegisterTableProcedureTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeRegisterTableProcedureTest.java index 35d26ef313fd..209a68e349a9 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeRegisterTableProcedureTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeRegisterTableProcedureTest.java @@ -23,6 +23,7 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; import java.util.Optional; import java.util.regex.Matcher; @@ -35,8 +36,10 @@ import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; @TestInstance(PER_CLASS) +@Execution(CONCURRENT) public abstract class BaseDeltaLakeRegisterTableProcedureTest extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreViewsTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreViewsTest.java index 9ebd6d489e20..aa0246a1fc17 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreViewsTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreViewsTest.java @@ -24,6 +24,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; import java.io.IOException; import java.nio.file.Path; @@ -36,11 +37,13 @@ import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; /** * Tests querying views on a schema which has a mix of Hive and Delta Lake tables. */ @TestInstance(PER_CLASS) +@Execution(CONCURRENT) public abstract class BaseDeltaLakeSharedMetastoreViewsTest extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreWithTableRedirectionsTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreWithTableRedirectionsTest.java index 6a000a020ffe..f63af5554e0b 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreWithTableRedirectionsTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreWithTableRedirectionsTest.java @@ -15,13 +15,10 @@ import io.trino.testing.AbstractTestQueryFramework; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import static io.trino.testing.TestingNames.randomNameSuffix; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public abstract class BaseDeltaLakeSharedMetastoreWithTableRedirectionsTest extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAdlsConnectorSmokeTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAdlsConnectorSmokeTest.java index f92500a0add7..c31f629d793f 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAdlsConnectorSmokeTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAdlsConnectorSmokeTest.java @@ -27,7 +27,6 @@ import io.trino.testing.QueryRunner; import okhttp3.OkHttpClient; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.TestInstance; import java.io.IOException; import java.io.UncheckedIOException; @@ -51,10 +50,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.regex.Matcher.quoteReplacement; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; import static org.testcontainers.containers.Network.newNetwork; -@TestInstance(PER_CLASS) public class TestDeltaLakeAdlsConnectorSmokeTest extends BaseDeltaLakeConnectorSmokeTest { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAdlsStorage.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAdlsStorage.java index 2695a163bb0a..db42eeb27bff 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAdlsStorage.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAdlsStorage.java @@ -22,7 +22,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import org.testcontainers.containers.Network; import java.nio.file.Files; @@ -41,9 +40,7 @@ import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.UUID.randomUUID; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeAdlsStorage extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java index 2eaa59bed38e..eaee1a13b3ee 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java @@ -20,15 +20,12 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder; import static io.trino.plugin.hive.containers.HiveHadoop.HIVE3_IMAGE; import static io.trino.testing.TestingNames.randomNameSuffix; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeFlushMetadataCacheProcedure extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeGcsConnectorSmokeTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeGcsConnectorSmokeTest.java index 33b68ba8bafa..9a5257625f94 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeGcsConnectorSmokeTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeGcsConnectorSmokeTest.java @@ -27,8 +27,6 @@ import io.trino.spi.security.ConnectorIdentity; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; import java.io.FileNotFoundException; import java.io.IOException; @@ -49,8 +47,6 @@ import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.regex.Matcher.quoteReplacement; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; import static org.testcontainers.containers.Network.newNetwork; /** @@ -59,8 +55,6 @@ * - gcp-credentials-key: A base64 encoded copy of the JSON authentication file for the service account used to connect to GCP. * For example, `cat service-account-key.json | base64` */ -@TestInstance(PER_CLASS) -@Execution(SAME_THREAD) public class TestDeltaLakeGcsConnectorSmokeTest extends BaseDeltaLakeConnectorSmokeTest { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeLocalConcurrentWritesTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeLocalConcurrentWritesTest.java index a687f994adec..1c011a1ec462 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeLocalConcurrentWritesTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeLocalConcurrentWritesTest.java @@ -24,7 +24,6 @@ import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.IOException; import java.io.UncheckedIOException; @@ -48,9 +47,7 @@ import static java.util.stream.Collectors.joining; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeLocalConcurrentWritesTest extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePartitioning.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePartitioning.java index aa03e0929781..a4a8cb2d3506 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePartitioning.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePartitioning.java @@ -17,14 +17,11 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakePartitioning extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeProjectionPushdownPlans.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeProjectionPushdownPlans.java index 69bb1df51727..e711a4ee035a 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeProjectionPushdownPlans.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeProjectionPushdownPlans.java @@ -42,7 +42,6 @@ import io.trino.testing.PlanTester; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.IOException; import java.io.UncheckedIOException; @@ -73,9 +72,7 @@ import static io.trino.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeProjectionPushdownPlans extends BasePushdownPlanTest { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeReadTimestamps.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeReadTimestamps.java index c0d4e1831c57..4c3cfcfc75cd 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeReadTimestamps.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeReadTimestamps.java @@ -20,7 +20,6 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.time.Instant; import java.time.LocalDateTime; @@ -46,9 +45,7 @@ import static java.time.temporal.ChronoField.MONTH_OF_YEAR; import static java.time.temporal.ChronoField.YEAR; import static java.util.stream.Collectors.joining; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeReadTimestamps extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedFileMetastoreWithTableRedirections.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedFileMetastoreWithTableRedirections.java index ccb389fc51f2..6d57ddf3cc4e 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedFileMetastoreWithTableRedirections.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedFileMetastoreWithTableRedirections.java @@ -19,7 +19,6 @@ import io.trino.testing.DistributedQueryRunner; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.TestInstance; import java.nio.file.Path; import java.util.Map; @@ -27,9 +26,7 @@ import static io.trino.plugin.deltalake.DeltaLakeConnectorFactory.CONNECTOR_NAME; import static io.trino.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeSharedFileMetastoreWithTableRedirections extends BaseDeltaLakeSharedMetastoreWithTableRedirectionsTest { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedGlueMetastoreWithTableRedirections.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedGlueMetastoreWithTableRedirections.java index 4bf2c695bed2..28fdc49fac83 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedGlueMetastoreWithTableRedirections.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedGlueMetastoreWithTableRedirections.java @@ -20,14 +20,12 @@ import io.trino.testing.DistributedQueryRunner; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.TestInstance; import java.nio.file.Path; import static io.trino.plugin.hive.metastore.glue.TestingGlueHiveMetastore.createTestingGlueHiveMetastore; import static io.trino.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; /** * Tests metadata operations on a schema which has a mix of Hive and Delta Lake tables. @@ -35,7 +33,6 @@ * Requires AWS credentials, which can be provided any way supported by the DefaultProviderChain * See https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default */ -@TestInstance(PER_CLASS) public class TestDeltaLakeSharedGlueMetastoreWithTableRedirections extends BaseDeltaLakeSharedMetastoreWithTableRedirectionsTest { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java index bef79413f6fd..4f2d01c36a3a 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java @@ -20,7 +20,6 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import static io.airlift.testing.Closeables.closeAllSuppress; import static io.trino.testing.TestingNames.randomNameSuffix; @@ -28,9 +27,7 @@ import static io.trino.testing.containers.Minio.MINIO_REGION; import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeSharedHiveMetastoreWithViews extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableStatistics.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableStatistics.java index 8215180aa70d..9b931b513490 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableStatistics.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableStatistics.java @@ -18,13 +18,10 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeTableStatistics extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java index 8deee1b2a95e..84d946cf8a64 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java @@ -16,7 +16,6 @@ import io.trino.plugin.hive.metastore.glue.GlueHiveMetastore; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.TestInstance; import java.nio.file.Files; import java.nio.file.Path; @@ -26,9 +25,7 @@ import static io.trino.plugin.hive.metastore.glue.TestingGlueHiveMetastore.createTestingGlueHiveMetastore; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.util.Objects.requireNonNull; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeTableWithCustomLocationUsingGlueMetastore extends BaseDeltaLakeTableWithCustomLocation { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestSplitPruning.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestSplitPruning.java index fe3acd47323a..728da7fec1c2 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestSplitPruning.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestSplitPruning.java @@ -23,7 +23,6 @@ import org.intellij.lang.annotations.Language; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.util.Arrays; import java.util.List; @@ -33,9 +32,7 @@ import static com.google.common.collect.MoreCollectors.onlyElement; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestSplitPruning extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeConcurrentModificationGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeConcurrentModificationGlueMetastore.java index c44b3865ff07..04debf6c3635 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeConcurrentModificationGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeConcurrentModificationGlueMetastore.java @@ -28,7 +28,6 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import software.amazon.awssdk.services.glue.GlueClient; import software.amazon.awssdk.services.glue.model.ConcurrentModificationException; @@ -48,9 +47,7 @@ import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.testing.TestingSession.testSessionBuilder; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeConcurrentModificationGlueMetastore extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeRenameToWithGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeRenameToWithGlueMetastore.java index 3e7d7e8fd5df..91aaa5b7123a 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeRenameToWithGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeRenameToWithGlueMetastore.java @@ -19,7 +19,6 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.nio.file.Files; import java.nio.file.Path; @@ -28,9 +27,7 @@ import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeRenameToWithGlueMetastore extends AbstractTestQueryFramework { diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java index f5b694bbf34c..6769f78d9859 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java @@ -21,7 +21,6 @@ import io.trino.testing.sql.TestView; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.IOException; import java.nio.file.Files; @@ -33,9 +32,7 @@ import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; import static java.util.Objects.requireNonNull; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestDeltaLakeViewsGlueMetastore extends AbstractTestQueryFramework { diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestShowStats.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestShowStats.java index bc85cd6ac757..f39637a4dfce 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestShowStats.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestShowStats.java @@ -19,15 +19,12 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import static io.trino.SystemSessionProperties.PREFER_PARTIAL_AGGREGATION; import static io.trino.SystemSessionProperties.USE_PARTIAL_DISTINCT_LIMIT; import static io.trino.SystemSessionProperties.USE_PARTIAL_TOPN; import static io.trino.tpch.TpchTable.NATION; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestShowStats extends AbstractTestQueryFramework { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java index 8022e1f20e4d..5718d81ca6b3 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java @@ -32,7 +32,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; import java.time.ZonedDateTime; @@ -60,9 +59,7 @@ import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public abstract class BaseIcebergConnectorSmokeTest extends BaseConnectorSmokeTest { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java index b1eac0baa3bf..e0f8f555cd1b 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java @@ -160,8 +160,6 @@ import static org.junit.jupiter.api.Assumptions.abort; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@Isolated // TODO remove -@TestInstance(PER_CLASS) public abstract class BaseIcebergConnectorTest extends BaseConnectorTest { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java index 20f934795c5a..ca0e34d2203d 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java @@ -21,7 +21,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.time.LocalDate; import java.util.ArrayList; @@ -34,9 +33,7 @@ import static io.trino.testing.MaterializedResult.DEFAULT_PRECISION; import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public abstract class BaseIcebergSystemTables extends AbstractTestQueryFramework { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java index 9a4d7c9043cc..372dcf157663 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java @@ -20,7 +20,6 @@ import io.trino.testing.QueryRunner; import io.trino.testing.sql.TestTable; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.IOException; import java.io.UncheckedIOException; @@ -33,11 +32,9 @@ import static io.trino.tpch.TpchTable.REGION; import static org.apache.iceberg.FileFormat.ORC; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; // Redundant over TestIcebergOrcConnectorTest, but exists to exercise BaseConnectorSmokeTest // Some features like materialized views may be supported by Iceberg only. -@TestInstance(PER_CLASS) public class TestIcebergConnectorSmokeTest extends BaseIcebergConnectorSmokeTest { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergGcsConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergGcsConnectorSmokeTest.java index 857d3c8f6f83..d58d5e626d74 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergGcsConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergGcsConnectorSmokeTest.java @@ -24,7 +24,6 @@ import io.trino.testing.TestingConnectorBehavior; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.IOException; import java.io.UncheckedIOException; @@ -43,9 +42,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.iceberg.FileFormat.ORC; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestIcebergGcsConnectorSmokeTest extends BaseIcebergConnectorSmokeTest { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetadataListing.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetadataListing.java index 0a52597bc083..c48b35ce196e 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetadataListing.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetadataListing.java @@ -26,7 +26,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.File; import java.util.Optional; @@ -35,9 +34,7 @@ import static io.trino.spi.security.SelectedRole.Type.ROLE; import static io.trino.testing.TestingSession.testSessionBuilder; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestIcebergMetadataListing extends AbstractTestQueryFramework { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergReadVersionedTable.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergReadVersionedTable.java index be0da7ff6a03..f8c2347f6d71 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergReadVersionedTable.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergReadVersionedTable.java @@ -17,7 +17,6 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.time.Instant; import java.time.ZonedDateTime; @@ -25,9 +24,7 @@ import static java.lang.String.format; import static java.time.ZoneOffset.UTC; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestIcebergReadVersionedTable extends AbstractTestQueryFramework { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergReadVersionedTableByTemporal.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergReadVersionedTableByTemporal.java index d7be4ed81fe5..d08b753ce4b8 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergReadVersionedTableByTemporal.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergReadVersionedTableByTemporal.java @@ -21,7 +21,6 @@ import io.trino.testing.containers.Minio; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import static io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; import static io.trino.testing.TestingNames.randomNameSuffix; @@ -30,9 +29,7 @@ import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestIcebergReadVersionedTableByTemporal extends AbstractTestQueryFramework { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java index 4db40a718a06..043ac3274ab8 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java @@ -54,7 +54,6 @@ import org.apache.iceberg.types.Types; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; import java.io.File; @@ -79,9 +78,7 @@ import static io.trino.tpch.TpchTable.NATION; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestIcebergSplitSource extends AbstractTestQueryFramework { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithExternalLocation.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithExternalLocation.java index 78155803885c..0bdac0f9bee1 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithExternalLocation.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithExternalLocation.java @@ -23,7 +23,6 @@ import io.trino.testing.QueryRunner; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.File; import java.io.IOException; @@ -37,9 +36,7 @@ import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestIcebergTableWithExternalLocation extends AbstractTestQueryFramework { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java index 9327478a493b..1366cee84cfe 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java @@ -63,7 +63,6 @@ import org.apache.iceberg.types.Types; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.Closeable; import java.nio.ByteBuffer; @@ -97,9 +96,7 @@ import static org.apache.iceberg.FileFormat.ORC; import static org.apache.iceberg.TableProperties.SPLIT_SIZE; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestIcebergV2 extends AbstractTestQueryFramework { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java index d4edc9ea8828..6ce46744358e 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java @@ -45,7 +45,6 @@ import org.apache.iceberg.FileFormat; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.util.List; @@ -59,14 +58,12 @@ import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; /* * TestIcebergGlueCatalogConnectorSmokeTest currently uses AWS Default Credential Provider Chain, * See https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default * on ways to set your AWS credentials which will be needed to run this test. */ -@TestInstance(PER_CLASS) public class TestIcebergGlueCatalogConnectorSmokeTest extends BaseIcebergConnectorSmokeTest { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java index 98e634b63a12..10bf1c9f0125 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java @@ -35,7 +35,6 @@ import org.apache.iceberg.io.FileIO; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.File; import java.nio.file.Files; @@ -56,14 +55,12 @@ import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; import static org.apache.iceberg.BaseMetastoreTableOperations.METADATA_LOCATION_PROP; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; /* * The test currently uses AWS Default Credential Provider Chain, * See https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default * on ways to set your AWS credentials which will be needed to run this test. */ -@TestInstance(PER_CLASS) public class TestIcebergGlueCatalogSkipArchive extends AbstractTestQueryFramework { diff --git a/plugin/trino-thrift/src/test/java/io/trino/plugin/thrift/integration/TestThriftProjectionPushdown.java b/plugin/trino-thrift/src/test/java/io/trino/plugin/thrift/integration/TestThriftProjectionPushdown.java index f16c0a6afe76..f4332e188787 100644 --- a/plugin/trino-thrift/src/test/java/io/trino/plugin/thrift/integration/TestThriftProjectionPushdown.java +++ b/plugin/trino-thrift/src/test/java/io/trino/plugin/thrift/integration/TestThriftProjectionPushdown.java @@ -38,7 +38,6 @@ import io.trino.testing.PlanTester; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.io.IOException; import java.util.Map; @@ -55,9 +54,7 @@ import static io.trino.testing.TestingHandles.TEST_CATALOG_NAME; import static io.trino.testing.TestingSession.testSessionBuilder; import static java.util.stream.Collectors.joining; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestThriftProjectionPushdown extends BaseRuleTest { diff --git a/testing/trino-tests/src/test/java/io/trino/tests/TestQueryPlanDeterminism.java b/testing/trino-tests/src/test/java/io/trino/tests/TestQueryPlanDeterminism.java index 28ed9a58c06f..2b429f189086 100644 --- a/testing/trino-tests/src/test/java/io/trino/tests/TestQueryPlanDeterminism.java +++ b/testing/trino-tests/src/test/java/io/trino/tests/TestQueryPlanDeterminism.java @@ -28,7 +28,6 @@ import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import java.util.List; @@ -36,9 +35,7 @@ import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static io.trino.testing.CustomFunctionBundle.CUSTOM_FUNCTIONS; import static io.trino.testing.TestingSession.testSessionBuilder; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -@TestInstance(PER_CLASS) public class TestQueryPlanDeterminism extends AbstractTestQueries { From 7b912beb33875f250251306f2ce09e286571e171 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Mon, 15 Apr 2024 16:31:35 +0200 Subject: [PATCH 2/3] Rename interface to accomodate non-partitioned data --- .../operator/function/EmptyTableFunctionPartition.java | 2 +- .../operator/function/RegularTableFunctionPartition.java | 2 +- ...{TableFunctionPartition.java => TableFunctionInput.java} | 2 +- .../io/trino/operator/function/TableFunctionOperator.java | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) rename core/trino-main/src/main/java/io/trino/operator/function/{TableFunctionPartition.java => TableFunctionInput.java} (94%) diff --git a/core/trino-main/src/main/java/io/trino/operator/function/EmptyTableFunctionPartition.java b/core/trino-main/src/main/java/io/trino/operator/function/EmptyTableFunctionPartition.java index 0661a0abab5d..169184192394 100644 --- a/core/trino-main/src/main/java/io/trino/operator/function/EmptyTableFunctionPartition.java +++ b/core/trino-main/src/main/java/io/trino/operator/function/EmptyTableFunctionPartition.java @@ -41,7 +41,7 @@ * a table function having KEEP WHEN EMPTY property must have single distribution. */ public class EmptyTableFunctionPartition - implements TableFunctionPartition + implements TableFunctionInput { private final TableFunctionDataProcessor tableFunction; private final int properChannelsCount; diff --git a/core/trino-main/src/main/java/io/trino/operator/function/RegularTableFunctionPartition.java b/core/trino-main/src/main/java/io/trino/operator/function/RegularTableFunctionPartition.java index 5b7c57f47877..fe3b1b035912 100644 --- a/core/trino-main/src/main/java/io/trino/operator/function/RegularTableFunctionPartition.java +++ b/core/trino-main/src/main/java/io/trino/operator/function/RegularTableFunctionPartition.java @@ -52,7 +52,7 @@ import static java.util.Objects.requireNonNull; public class RegularTableFunctionPartition - implements TableFunctionPartition + implements TableFunctionInput { private final PagesIndex pagesIndex; private final int partitionStart; diff --git a/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionPartition.java b/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionInput.java similarity index 94% rename from core/trino-main/src/main/java/io/trino/operator/function/TableFunctionPartition.java rename to core/trino-main/src/main/java/io/trino/operator/function/TableFunctionInput.java index 88375ae87e81..84e18c7da6e2 100644 --- a/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionPartition.java +++ b/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionInput.java @@ -16,7 +16,7 @@ import io.trino.operator.WorkProcessor; import io.trino.spi.Page; -public interface TableFunctionPartition +public interface TableFunctionInput { WorkProcessor toOutputPages(); } diff --git a/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionOperator.java b/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionOperator.java index 0143ff022f6f..9c9b007bcc77 100644 --- a/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionOperator.java @@ -299,7 +299,7 @@ public TableFunctionOperator( markerChannels, passThroughSpecifications, processEmptyInput)) - .flatMap(TableFunctionPartition::toOutputPages); + .flatMap(TableFunctionInput::toOutputPages); } @Override @@ -530,7 +530,7 @@ private static int findGroupEnd(PagesIndex pagesIndex, PagesHashStrategy pagesHa return findEndPosition(startPosition, pagesIndex.getPositionCount(), (firstPosition, secondPosition) -> pagesIndex.positionIdenticalToPosition(pagesHashStrategy, firstPosition, secondPosition)); } - private WorkProcessor pagesIndexToTableFunctionPartitions( + private WorkProcessor pagesIndexToTableFunctionPartitions( PagesIndex pagesIndex, HashStrategies hashStrategies, TableFunctionProcessorProvider tableFunctionProvider, @@ -553,7 +553,7 @@ private WorkProcessor pagesIndexToTableFunctionPartition private boolean processEmpty = processEmptyInput; @Override - public WorkProcessor.ProcessState process() + public WorkProcessor.ProcessState process() { if (partitionStart == pagesIndex.getPositionCount()) { if (processEmpty && pagesIndex.getPositionCount() == 0) { From a8928205d9db7e34292b3f241a2b22e25831541c Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Wed, 3 Apr 2024 21:38:23 +0200 Subject: [PATCH 3/3] Fix table function execution without partitioning Previously, when table function did not declare partitioning, it would be globally distributed, but on a worker node it would run single-threaded and first buffer all data in memory, like a one big WINDOW. After the change, the local execution processes input pages in a streaming fashion. This commit also fixes property derivations for a case where table function is partitioned on empty list of symbols (global grouping). --- .../io/trino/operator/WorkProcessorUtils.java | 6 +- .../function/StreamTableFunctionInput.java | 125 ++++++++++++++++++ .../function/TableFunctionOperator.java | 88 ++++++++---- .../sql/planner/LocalExecutionPlanner.java | 5 +- 4 files changed, 189 insertions(+), 35 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/operator/function/StreamTableFunctionInput.java diff --git a/core/trino-main/src/main/java/io/trino/operator/WorkProcessorUtils.java b/core/trino-main/src/main/java/io/trino/operator/WorkProcessorUtils.java index 16c6aa33fadf..acabf804aa1d 100644 --- a/core/trino-main/src/main/java/io/trino/operator/WorkProcessorUtils.java +++ b/core/trino-main/src/main/java/io/trino/operator/WorkProcessorUtils.java @@ -38,7 +38,7 @@ public final class WorkProcessorUtils { private WorkProcessorUtils() {} - static Iterator iteratorFrom(WorkProcessor processor) + public static Iterator iteratorFrom(WorkProcessor processor) { requireNonNull(processor, "processor is null"); return new AbstractIterator<>() @@ -58,7 +58,7 @@ protected T computeNext() }; } - static Iterator> yieldingIteratorFrom(WorkProcessor processor) + public static Iterator> yieldingIteratorFrom(WorkProcessor processor) { return new YieldingIterator<>(processor); } @@ -95,7 +95,7 @@ protected Optional computeNext() } } - static WorkProcessor fromIterator(Iterator iterator) + public static WorkProcessor fromIterator(Iterator iterator) { requireNonNull(iterator, "iterator is null"); return create(() -> { diff --git a/core/trino-main/src/main/java/io/trino/operator/function/StreamTableFunctionInput.java b/core/trino-main/src/main/java/io/trino/operator/function/StreamTableFunctionInput.java new file mode 100644 index 000000000000..3d3b6339efb7 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/function/StreamTableFunctionInput.java @@ -0,0 +1,125 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.function; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.operator.PagesIndex; +import io.trino.operator.WorkProcessor; +import io.trino.operator.WorkProcessor.ProcessState; +import io.trino.spi.Page; +import io.trino.spi.function.table.TableFunctionDataProcessor; +import io.trino.spi.function.table.TableFunctionProcessorState; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.concurrent.MoreFutures.toListenableFuture; +import static io.trino.operator.WorkProcessorUtils.yieldingIteratorFrom; +import static java.util.Objects.requireNonNull; + +public class StreamTableFunctionInput + implements WorkProcessor.Process> +{ + private final TableFunctionDataProcessor tableFunction; + private final int properChannelsCount; + private final int passThroughSourcesCount; + private final List> requiredChannels; + private final Optional> markerChannels; + private final List passThroughSpecifications; + + private final Iterator> inputPages; + private final PagesIndex pagesIndex; + private boolean finished; + + public StreamTableFunctionInput( + TableFunctionDataProcessor tableFunction, + int properChannelsCount, + int passThroughSourcesCount, + List> requiredChannels, + Optional> markerChannels, + List passThroughSpecifications, + WorkProcessor inputPages, + PagesIndex pagesIndex) + { + this.tableFunction = requireNonNull(tableFunction, "tableFunction is null"); + this.properChannelsCount = properChannelsCount; + this.passThroughSourcesCount = passThroughSourcesCount; + this.requiredChannels = requiredChannels.stream() + .map(ImmutableList::copyOf) + .collect(toImmutableList()); + this.markerChannels = markerChannels.map(ImmutableMap::copyOf); + this.passThroughSpecifications = ImmutableList.copyOf(requireNonNull(passThroughSpecifications, "passThroughSpecifications is null")); + this.inputPages = yieldingIteratorFrom(requireNonNull(inputPages, "inputPages is null")); + this.pagesIndex = requireNonNull(pagesIndex, "pagesIndex is null"); + } + + @Override + public ProcessState> process() + { + if (finished) { + return ProcessState.finished(); + } + + if (inputPages.hasNext()) { + Optional next = inputPages.next(); + if (next.isEmpty()) { + return ProcessState.yielded(); + } + Page currentInputPage = next.get(); + pagesIndex.clear(); + pagesIndex.addPage(currentInputPage); + return ProcessState.ofResult(new RegularTableFunctionPartition( + pagesIndex, + 0, + pagesIndex.getPositionCount(), + new TableFunctionDataProcessor() + { + @Override + public TableFunctionProcessorState process(@org.jetbrains.annotations.Nullable List> input) + { + if (input == null) { + // end of page + return TableFunctionProcessorState.Finished.FINISHED; + } + TableFunctionProcessorState process = tableFunction.process(input); + if (process instanceof TableFunctionProcessorState.Finished) { + finished = true; + } + return process; + } + }, + properChannelsCount, + passThroughSourcesCount, + requiredChannels, + markerChannels, + passThroughSpecifications) + .toOutputPages()); + } + + // finish + finished = true; + return ProcessState.ofResult(WorkProcessor.create(() -> { + TableFunctionProcessorState state = tableFunction.process(null); + return switch (state) { + case TableFunctionProcessorState.Processed processed -> ProcessState.ofResult(processed.getResult()); + case TableFunctionProcessorState.Blocked blocked -> ProcessState.blocked(toListenableFuture(blocked.getFuture())); + case TableFunctionProcessorState.Finished __ -> ProcessState.finished(); + }; + })); + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionOperator.java b/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionOperator.java index 9c9b007bcc77..7040380eabf8 100644 --- a/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/function/TableFunctionOperator.java @@ -52,6 +52,7 @@ import static io.trino.spi.connector.SortOrder.ASC_NULLS_LAST; import static java.util.Collections.nCopies; import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; public class TableFunctionOperator implements Operator @@ -93,7 +94,7 @@ public static class TableFunctionOperatorFactory private final boolean pruneWhenEmpty; // partitioning channels from all sources - private final List partitionChannels; + private final Optional> partitionChannels; // subset of partition channels that are already grouped private final List prePartitionedChannels; @@ -125,7 +126,7 @@ public TableFunctionOperatorFactory( Optional> markerChannels, List passThroughSpecifications, boolean pruneWhenEmpty, - List partitionChannels, + Optional> partitionChannels, List prePartitionedChannels, List sortChannels, List sortOrders, @@ -142,12 +143,20 @@ public TableFunctionOperatorFactory( requireNonNull(passThroughSpecifications, "passThroughSpecifications is null"); requireNonNull(partitionChannels, "partitionChannels is null"); requireNonNull(prePartitionedChannels, "prePartitionedChannels is null"); - checkArgument(partitionChannels.containsAll(prePartitionedChannels), "prePartitionedChannels must be a subset of partitionChannels"); requireNonNull(sortChannels, "sortChannels is null"); requireNonNull(sortOrders, "sortOrders is null"); - checkArgument(sortChannels.size() == sortOrders.size(), "The number of sort channels must be equal to the number of sort orders"); - checkArgument(preSortedPrefix <= sortChannels.size(), "The number of pre-sorted channels must be lower or equal to the number of sort channels"); - checkArgument(preSortedPrefix == 0 || ImmutableSet.copyOf(prePartitionedChannels).equals(ImmutableSet.copyOf(partitionChannels)), "preSortedPrefix can only be greater than zero if all partition channels are pre-grouped"); + if (partitionChannels.isPresent()) { + checkArgument(partitionChannels.get().containsAll(prePartitionedChannels), "prePartitionedChannels must be a subset of partitionChannels"); + checkArgument(sortChannels.size() == sortOrders.size(), "The number of sort channels must be equal to the number of sort orders"); + checkArgument(preSortedPrefix <= sortChannels.size(), "The number of pre-sorted channels must be lower or equal to the number of sort channels"); + checkArgument(preSortedPrefix == 0 || ImmutableSet.copyOf(prePartitionedChannels).equals(ImmutableSet.copyOf(partitionChannels.get())), "preSortedPrefix can only be greater than zero if all partition channels are pre-grouped"); + } + else { + checkArgument(prePartitionedChannels.isEmpty(), "prePartitionedChannels must be empty when partitionChannels is absent"); + checkArgument(sortChannels.isEmpty(), "sortChannels must be empty when partitionChannels is absent"); + checkArgument(sortOrders.isEmpty(), "sortOrders must be empty when partitionChannels is absent"); + checkArgument(preSortedPrefix == 0, "preSortedPrefix must be zero when partitionChannels is absent"); + } requireNonNull(sourceTypes, "sourceTypes is null"); requireNonNull(pagesIndexFactory, "pagesIndexFactory is null"); @@ -164,7 +173,7 @@ public TableFunctionOperatorFactory( this.markerChannels = markerChannels.map(ImmutableMap::copyOf); this.passThroughSpecifications = ImmutableList.copyOf(passThroughSpecifications); this.pruneWhenEmpty = pruneWhenEmpty; - this.partitionChannels = ImmutableList.copyOf(partitionChannels); + this.partitionChannels = partitionChannels.map(ImmutableList::copyOf); this.prePartitionedChannels = ImmutableList.copyOf(prePartitionedChannels); this.sortChannels = ImmutableList.copyOf(sortChannels); this.sortOrders = ImmutableList.copyOf(sortOrders); @@ -250,7 +259,7 @@ public TableFunctionOperator( Optional> markerChannels, List passThroughSpecifications, boolean pruneWhenEmpty, - List partitionChannels, + Optional> partitionChannels, List prePartitionedChannels, List sortChannels, List sortOrders, @@ -268,12 +277,20 @@ public TableFunctionOperator( requireNonNull(passThroughSpecifications, "passThroughSpecifications is null"); requireNonNull(partitionChannels, "partitionChannels is null"); requireNonNull(prePartitionedChannels, "prePartitionedChannels is null"); - checkArgument(partitionChannels.containsAll(prePartitionedChannels), "prePartitionedChannels must be a subset of partitionChannels"); requireNonNull(sortChannels, "sortChannels is null"); requireNonNull(sortOrders, "sortOrders is null"); - checkArgument(sortChannels.size() == sortOrders.size(), "The number of sort channels must be equal to the number of sort orders"); - checkArgument(preSortedPrefix <= sortChannels.size(), "The number of pre-sorted channels must be lower or equal to the number of sort channels"); - checkArgument(preSortedPrefix == 0 || ImmutableSet.copyOf(prePartitionedChannels).equals(ImmutableSet.copyOf(partitionChannels)), "preSortedPrefix can only be greater than zero if all partition channels are pre-grouped"); + if (partitionChannels.isPresent()) { + checkArgument(partitionChannels.get().containsAll(prePartitionedChannels), "prePartitionedChannels must be a subset of partitionChannels"); + checkArgument(sortChannels.size() == sortOrders.size(), "The number of sort channels must be equal to the number of sort orders"); + checkArgument(preSortedPrefix <= sortChannels.size(), "The number of pre-sorted channels must be lower or equal to the number of sort channels"); + checkArgument(preSortedPrefix == 0 || ImmutableSet.copyOf(prePartitionedChannels).equals(ImmutableSet.copyOf(partitionChannels.get())), "preSortedPrefix can only be greater than zero if all partition channels are pre-grouped"); + } + else { + checkArgument(prePartitionedChannels.isEmpty(), "prePartitionedChannels must be empty when partitionChannels is absent"); + checkArgument(sortChannels.isEmpty(), "sortChannels must be empty when partitionChannels is absent"); + checkArgument(sortOrders.isEmpty(), "sortOrders must be empty when partitionChannels is absent"); + checkArgument(preSortedPrefix == 0, "preSortedPrefix must be zero when partitionChannels is absent"); + } requireNonNull(sourceTypes, "sourceTypes is null"); requireNonNull(pagesIndexFactory, "pagesIndexFactory is null"); @@ -283,23 +300,36 @@ public TableFunctionOperator( this.processEmptyInput = !pruneWhenEmpty; PagesIndex pagesIndex = pagesIndexFactory.newPagesIndex(sourceTypes, expectedPositions); - HashStrategies hashStrategies = new HashStrategies(pagesIndex, partitionChannels, prePartitionedChannels, sortChannels, sortOrders, preSortedPrefix); - - this.outputPages = pageBuffer.pages() - .transform(new PartitionAndSort(pagesIndex, hashStrategies, processEmptyInput)) - .flatMap(groupPagesIndex -> pagesIndexToTableFunctionPartitions( - groupPagesIndex, - hashStrategies, - tableFunctionProvider, - session, - functionHandle, - properChannelsCount, - passThroughSourcesCount, - requiredChannels, - markerChannels, - passThroughSpecifications, - processEmptyInput)) - .flatMap(TableFunctionInput::toOutputPages); + if (partitionChannels.isEmpty()) { + this.outputPages = WorkProcessor.create(new StreamTableFunctionInput( + tableFunctionProvider.getDataProcessor(session, functionHandle), + properChannelsCount, + passThroughSourcesCount, + requiredChannels, + markerChannels, + passThroughSpecifications, + pageBuffer.pages(), + pagesIndex)) + .flatMap(identity()); + } + else { + HashStrategies hashStrategies = new HashStrategies(pagesIndex, partitionChannels.get(), prePartitionedChannels, sortChannels, sortOrders, preSortedPrefix); + this.outputPages = pageBuffer.pages() + .transform(new PartitionAndSort(pagesIndex, hashStrategies, processEmptyInput)) + .flatMap(groupPagesIndex -> pagesIndexToTableFunctionPartitions( + groupPagesIndex, + hashStrategies, + tableFunctionProvider, + session, + functionHandle, + properChannelsCount, + passThroughSourcesCount, + requiredChannels, + markerChannels, + passThroughSpecifications, + processEmptyInput)) + .flatMap(TableFunctionInput::toOutputPages); + } } @Override diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java index 896bbe52e204..69aa3e3fcbf6 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java @@ -1660,10 +1660,9 @@ public PhysicalOperation visitTableFunctionProcessor(TableFunctionProcessorNode } } - List partitionChannels = node.getSpecification() + Optional> partitionChannels = node.getSpecification() .map(DataOrganizationSpecification::partitionBy) - .map(list -> getChannelsForSymbols(list, source.getLayout())) - .orElse(ImmutableList.of()); + .map(list -> getChannelsForSymbols(list, source.getLayout())); List sortChannels = ImmutableList.of(); List sortOrders = ImmutableList.of();